Handle xdg popups

Signed-off-by: Schmarni <marnistromer@gmail.com>

.github/workflows/build.yml

@@ -1,46 +1,26 @@
-name: Build
+name: CI
 
 on:
   push:
     branches:
-      - '*'
+      - "*"
 
 jobs:
-  build_and_package:
+  check:
     runs-on: ubuntu-latest
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Install runtime dependencies
-        run: sudo apt install -y --no-install-recommends libxkbcommon-dev libstdc++6 libopenxr-dev libx11-dev libxfixes-dev libgl1-mesa-dev libegl1-mesa-dev libgbm-dev libfontconfig-dev libjsoncpp-dev  libxcb1-dev libglx-dev libxcb-glx0-dev libdrm-dev libwayland-dev libfreetype-dev libpng-dev
-
-      - name: Install build dependencies
-        run: sudo apt install -y --no-install-recommends cmake ninja-build
+        run: sudo apt install -y --no-install-recommends libopenxr-dev libwayland-dev libasound2-dev
 
       - name: Set up Rust
         uses: actions-rs/toolchain@v1
         with:
-          toolchain: stable
+          toolchain: 1.88.0
+          override: true
 
       - name: Build server
         run: cargo build --release
-
-
-      - name: Install appimagetool
-        run: |
-            wget https://github.com/AppImage/AppImageKit/releases/download/continuous/appimagetool-$(uname -m).AppImage -O /usr/local/bin/appimagetool; \
-            chmod +x /usr/local/bin/appimagetool; \
-            sed -i 's|AI\x02|\x00\x00\x00|' /usr/local/bin/appimagetool
-      - name: Install cargo-appimage
-        run: cargo install cargo-appimage
-
-      - name: Generate AppImage
-        run: cargo appimage
-
-      - name: Upload AppImage
-        uses: actions/upload-artifact@v2
-        with:
-          name: appimage
-          path: '*.AppImage'

.zed/debug.json

@@ -0,0 +1,94 @@
+[
+	{
+		"label": "Debug",
+		"adapter": "CodeLLDB",
+		"program": "target/debug/stardust-xr-server",
+		"args": [],
+		"env": {
+			"RUST_LOG": "debug"
+		},
+		"request": "launch",
+		"build": {
+			"command": "cargo",
+			"args": [
+				"build",
+				"--bin=stardust-xr-server",
+				"--package=stardust-xr-server"
+			]
+		}
+	},
+	{
+		"label": "Debug (Overlay)",
+		"adapter": "CodeLLDB",
+		"program": "target/debug/stardust-xr-server",
+		"args": ["-o", "1"],
+		"env": {
+			"RUST_LOG": "debug"
+		},
+		"request": "launch",
+		"build": {
+			"command": "cargo",
+			"args": [
+				"build",
+				"--bin=stardust-xr-server",
+				"--package=stardust-xr-server"
+			]
+		}
+	},
+	{
+		"label": "Debug (Headless)",
+		"adapter": "CodeLLDB",
+		"program": "target/debug/stardust-xr-server",
+		"args": [],
+		"env": {
+			"RUST_LOG": "debug",
+			"DISPLAY": "",
+			"WAYLAND_DISPLAY": ""
+		},
+		"request": "launch",
+		"build": {
+			"command": "cargo",
+			"args": [
+				"build",
+				"--bin=stardust-xr-server",
+				"--package=stardust-xr-server"
+			]
+		}
+	},
+	{
+		"label": "Debug (Flatscreen)",
+		"adapter": "CodeLLDB",
+		"program": "target/debug/stardust-xr-server",
+		"args": ["-f"],
+		"env": {
+			"RUST_LOG": "debug"
+		},
+		"request": "launch",
+		"build": {
+			"command": "cargo",
+			"args": [
+				"build",
+				"--bin=stardust-xr-server",
+				"--package=stardust-xr-server"
+			]
+		}
+	},
+	{
+		"label": "Debug (Flatscreen, Restore Latest)",
+		"adapter": "CodeLLDB",
+		"program": "target/debug/stardust-xr-server",
+		"args": ["-f", "--restore", "latest"],
+		"env": {
+			"RUST_LOG": "debug"
+		},
+		"request": "launch",
+		"build": {
+			"command": "cargo",
+			"args": [
+				"build",
+				"--bin=stardust-xr-server",
+				"--package=stardust-xr-server"
+			]
+		}
+	}
+]

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
-edition = "2021"
-rust-version = "1.75"
+edition = "2024"
+rust-version = "1.88"
 name = "stardust-xr-server"
 version = "0.45.0"
 authors = ["Nova King <technobaboo@proton.me>"]
@@ -14,6 +14,7 @@ members = ["codegen"]
 
 [workspace.dependencies.stardust-xr]
 git = "https://github.com/StardustXR/core.git"
+branch = "dev"
 
 [[bin]]
 name = "stardust-xr-server"
@@ -21,56 +22,65 @@ path = "src/main.rs"
 
 [features]
 default = ["wayland"]
-wayland = ["dep:smithay", "dep:wayland-scanner", "dep:wayland-backend"]
+wayland = [
+	"dep:cluFlock",
+	"dep:waynest",
+	"dep:tokio-stream",
+	"dep:memmap2",
+	"dep:drm-fourcc",
+	"dep:memfd",
+	"dep:vulkano",
+	"dep:wgpu-hal",
+	"dep:ash",
+]
 profile_tokio = ["dep:console-subscriber", "tokio/tracing"]
-profile_app = ["dep:tracing-tracy"]
-local_deps = ["stereokit-rust/force-local-deps"]
+profile_app = ["dep:tracing-tracy", "bevy/trace_tracy", "bevy/trace"]
+bevy_debugging = ["bevy/bevy_remote", "bevy/track_location"]
 
 [package.metadata.appimage]
 auto_link = true
-auto_link_exclude_list = [
-	"libc*",
-	"libdl*",
-	"libpthread*",
-	"ld-linux*",
-	"libGL*",
-	"libEGL*",
-]
+auto_link_exclude_list = ["libc*", "libdl*", "libpthread*", "ld-linux*"]
+
+
+[profile.release]
+lto = true
+strip = true
+opt-level = 3
 
 [profile.dev.package."*"]
 opt-level = 3
-debug = true
-strip = "none"
-debug-assertions = true
-overflow-checks = true
 
-[profile.release]
-opt-level = 3
-debug = "line-tables-only"
-strip = "none"
-debug-assertions = true
-overflow-checks = false
+[patch.crates-io]
+bevy_mod_openxr = { git = "https://github.com/awtterpip/bevy_oxr" }
+bevy_mod_xr = { git = "https://github.com/awtterpip/bevy_oxr" }
+bevy_gltf = { git = "https://github.com/Schmarni-Dev/bevy", branch = "gltf_backport" }
+# TODO: figure out how to not need this horrifing patch
+wgpu = { git = "https://github.com/Schmarni-Dev/wgpu", branch = "bad_dmabuf_workaround" }
+wgpu-types = { git = "https://github.com/Schmarni-Dev/wgpu", branch = "bad_dmabuf_workaround" }
+wgpu-core = { git = "https://github.com/Schmarni-Dev/wgpu", branch = "bad_dmabuf_workaround" }
+naga = { git = "https://github.com/Schmarni-Dev/wgpu", branch = "bad_dmabuf_workaround" }
+wgpu-hal = { git = "https://github.com/Schmarni-Dev/wgpu", branch = "bad_dmabuf_workaround" }
 
 [dependencies]
 # small utility thingys
-once_cell = "1.19.0"
 nanoid = "0.4.0"
 lazy_static = "1.5.0"
 rand = "0.8.5"
 rustc-hash = "2.0.0"
-portable-atomic = { version = "1.7.0", features = ["float", "std"] }
-send_wrapper = "0.6.0"
 slotmap = "1.0.7"
 global_counter = "=0.2.2"
 parking_lot = "0.12.3"
+dashmap = "6.1.0"
 
 # rust errors/logging
 color-eyre = { version = "0.6.3", default-features = false }
 clap = { version = "4.5.13", features = ["derive"] }
 console-subscriber = { version = "0.4.0", optional = true }
-tracing = "0.1.40"
+thiserror = "2.0.11"
+tracing = { version = "0.1.40", features = ["release_max_level_warn"] }
 tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
 tracing-tracy = { version = "0.11.1", optional = true }
+tracy-client = { version = "=0.18.0" }
 
 # (de)serialization
 serde = { version = "1.0.205", features = ["derive"] }
@@ -81,39 +91,75 @@ toml = "0.8.19"
 glam = { version = "0.29.0", features = ["mint", "serde"] }
 mint = "0.5.9"
 tokio = { version = "1.39.2", features = ["rt-multi-thread", "signal", "time"] }
-prisma = "0.1.1"
+
+# bevy
+bevy = { version = "0.16", default-features = false, features = [
+	"bevy_asset",
+	"bevy_audio",
+	"bevy_color",
+	"bevy_core_pipeline",
+	"bevy_gizmos",
+	"bevy_gltf",
+	"bevy_log",
+	"bevy_pbr",
+	"bevy_render",
+	"bevy_window",
+	"bevy_winit",
+	"std",
+	"x11",
+	"wayland",
+	"mp3",
+	"wav",
+	"qoi",
+	"png",
+	"hdr",
+	"jpeg",
+	"tonemapping_luts",
+	"multi_threaded",
+] }
+bevy_mod_xr = "0.3"
+bevy_mod_openxr = "0.3"
+# bevy_sk.git = "https://github.com/MalekiRe/bevy_sk"
+bevy_sk = { git = "https://github.com/technobaboo/bevy_sk", branch = "stochastic" }
+# bevy-dmabuf = "0.2.0"
+bevy-dmabuf.git = "https://github.com/Schmarni-Dev/bevy-dmabuf"
+
+# bevy-mesh-text-3d.git = "https://github.com/terhechte/bevy-mesh-text-3d"
+# use my fork until my pr to use minimal bevy features was merged
+bevy-mesh-text-3d.git = "https://github.com/Schmarni-Dev/bevy-mesh-text-3d"
+
+openxr = "0.19"
 
 # linux stuffs
-libc = "0.2.155"
-nix = "0.29.0"
 input-event-codes = "6.2.0"
-zbus = { version = "4.4.0", default-features = false, features = ["tokio"] }
+zbus = { version = "5.0.0", default-features = false, features = ["tokio"] }
 directories = "5.0.1"
 xkbcommon-rs = "0.1.0"
+cosmic-text = "0.14.2"
 
-# wayland
-wayland-backend = { version = "0.3.7", optional = true, default-features = false }
-wayland-scanner = { version = "0.31.4", optional = true }
-
-[dependencies.smithay]
-# git = "https://github.com/technobaboo/smithay.git"
-# git = "https://github.com/colinmarc/smithay.git"
-git = "https://github.com/smithay/smithay.git"
-# path = "../smithay"
-default-features = false
-features = ["desktop", "backend_drm", "renderer_gl", "wayland_frontend"]
-optional = true
-
-
-[dependencies.stereokit-rust]
-# path = "../StereoKit-rust"
-git = "https://github.com/mvvvv/StereoKit-rust.git"
-# git = "https://github.com/technobaboo/StereoKit-rust.git"
-features = ["no-event-loop"]
-default-features = false
+# Wayland
+cluFlock = { version = "1.2.7", optional = true } # for the lockfile checking
+# waynest = { git = "https://github.com/verdiwm/waynest.git", features = [
+# 	"server",
+# 	"stable",
+# 	"tracing",
+# ], default-features = false, optional = true }
+waynest = { git = "https://github.com/technobaboo/waynest.git", branch = "fix/wayland-drm", features = [
+	"server",
+	"stable",
+	"external",
+	"tracing",
+], default-features = false, optional = true }
+tokio-stream = { version = "0.1.17", optional = true }
+memmap2 = { version = "0.9.5", optional = true }
+drm-fourcc = { version = "2.2.0", optional = true }
+memfd = { version = "0.6.4", optional = true }
+vulkano = { git = "https://github.com/Schmarni-Dev/vulkano", branch = "0_35_dmabuf_fixes", optional = true }
+wgpu-hal = { version = "24", optional = true, features = ["vulkan"] }
+ash = { version = "0.38.0", optional = true, default-features = false }
+rustix = { version = "1.0.8", features = ["time"] }
 
 [dependencies.stardust-xr]
 workspace = true
-
 [dependencies.stardust-xr-server-codegen]
 path = "codegen"

README.md

@@ -4,96 +4,45 @@ Stardust XR is a display server for VR and AR headsets on Linux-based systems. [
 
 ![workflow](/img/workflow.png)
 
-## Core Dependencies 
-| **Dependency**              | **Ubuntu/Debian**                                                                               | **Arch Linux**                                    | **Fedora**                                                  |
-|-----------------------------|-------------------------------------------------------------------------------------------------|---------------------------------------------------|-------------------------------------------------------------|
-| **Cargo**                   | `cargo`                                                                                       | `cargo` | `cargo` |
-| **CMake**                   | `cmake`                                                                                       | `cmake`                                           | `cmake`                                                     |
-| **EGL+GLES 3.2**            | `libegl1-mesa-dev`, `libgles2-mesa-dev`                                                         | `mesa` *(provides EGL/GLES libraries and headers)* | `mesa-libEGL-devel`, `mesa-libGLES-devel`                     |
-| **GLX+Xlib**                | `libx11-dev`, `libxfixes-dev`, `libxcb1-dev`, `libgl1-mesa-dev`, `libxkbcommon-dev`              | `libx11`, `libxfixes`, `libxcb` *(and GLX via mesa)*| `libX11-devel`, `libXfixes-devel`, `libxcb-devel`, `mesa-libGL-devel` *(or equivalent)* |
-| **fontconfig**              | `libfontconfig1-dev`                                                                            | `fontconfig`                                      | `fontconfig-devel`                                          |
-| **dlopen** (glibc function) | Provided by `libc6-dev` (part of the core C library)                                            | Provided by `glibc` *(included in base-devel)*    | Provided by `glibc-devel`                                     |
-| **OpenXR Loader**           | `libopenxr-loader1`, `libopenxr-dev`, `libopenxr1-monado`                                       | `openxr`                                          | `openxr-devel`                                              |
-
 Command line installation of core & dynamic dependencies are provided below:
 <details>
 <summary>Ubuntu/Debian</summary>
   <pre><code class="language-bash">
-  sudo apt-get update && sudo apt-get install -y \
+  sudo apt update && sudo apt install \
   build-essential \
   cargo \
   cmake \
-  libegl1-mesa-dev libgles2-mesa-dev \
-  libx11-dev libxfixes-dev libxcb1-dev libxau-dev libgl1-mesa-dev libxkbcommon-dev \
-  libfontconfig1-dev libfreetype6-dev libharfbuzz-dev libgraphite2-dev \
-  libc6-dev \
-  libopenxr-loader1 libopenxr-dev libopenxr1-monado libwayland-dev \
-  libjsoncpp-dev libdrm-dev libexpat1-dev libxcb-randr0-dev \
-  libxml2-dev libffi-dev libbz2-dev libpng-dev libbrotli-dev liblzma-dev libglib2.0-dev libpcre2-dev
+  libxkbcommon-dev libudev1 libinput10 libcap2 libmtdev1 libevdev2 libwacom9 libgudev-1.0-0 \
+  libglib2.0-dev libffi8 libpcre2-dev libxkbcommon-x11-dev libxcb-dev libxcb-xkb-dev libxau-dev \ 
+  libstdc++-dev libx11-dev libxfixes-dev libegl-dev libgbm-dev libfontconfig1-dev libgl-dev \
+  libdrm-dev libexpat1-dev libfreetype6-dev libxml2-dev zlib1g-dev libbz2-dev libpng-dev \ 
+  libharfbuzz-dev libbrotli-dev liblzma-dev libraphite2-dev
   </code></pre>
 </details>
 
+<details>
+<summary>Fedora</summary>
+  <pre><code class="language-bash">
+  sudo apt update && sudo apt install \
+  cargo \
+  cmake \
+  libxkbcommon-devel systemd-devel libinput-devel libcap-devel mtdev-devel libevdev-devel glib2-devel \
+  libffi-devel pcre2-devel libxkbcommon-x11-devel libxcb-devel libXau-devel libstdc++-devel libx11-devel libxfixes-devel \
+  mesa-libEGL-devel mesa-libgbm-devel fontconfig-devel libdrm-devel expat-devel freetype-devel libxml2-devel zlib-devel \
+  bzip2-devel libpng-devel harfbuzz-devel brotli-devel xz-devel graphite2-devel
+  </code></pre>
+</details>
+
+
 <details>
 <summary>Arch Linux</summary>
   <pre><code class="language-bash">
   sudo pacman -Syu --needed \
-  base-devel \
-  rust \
-  cmake \
-  mesa \
-  libx11 \
-  libxfixes \
-  libxcb \
-  libxkbcommon \
-  fontconfig \
-  freetype2 \
-  openxr \
-  jsoncpp \
-  libffi \
-  wayland \
-  expat \
-  libxml2 \
-  libxau \
-  bzip2 \
-  xz \
-  libpng \
-  brotli \
-  pcre2 \
-  glib2 \
-  libdrm
-  </code></pre>
-</details>
-<details>
-<summary>Fedora</summary> 
-  <pre><code class="language-bash">
-sudo dnf group install development-tools && \
-sudo dnf install -y \
   cargo \
   cmake \
-  mesa-libEGL-devel \
-  mesa-libGLES-devel \
-  libX11-devel \
-  libXfixes-devel \
-  libxcb-devel \
-  libxkbcommon-devel \
-  fontconfig-devel \
-  freetype-devel \
-  harfbuzz-devel \
-  graphite2-devel \
-  openxr-devel \ 
-  wayland-devel \
-  jsoncpp-devel \
-  libdrm-devel \
-  expat-devel \
-  xcb-util-devel \
-  libxml2-devel \
-  libXau-devel \
-  bzip2-devel \
-  xz-devel \
-  libpng-devel \
-  brotli-devel \
-  pcre2-devel \
-  glib2-devel
+  libxkbcommon systemd libinput libcap mtdev libevdev libwacom glib2 libffi pcre2 libxkbcommon-x11 \
+  libxcb libxau libx11 libxfixes mesa fontconfig libdrm expat freetype2 libxml2 zlib bzip2 \ 
+  libpng harfbuzz brotli xz graphite
   </code></pre>
 </details>
 
@@ -118,7 +67,7 @@ We've provided a manual installation script [here](https://github.com/cybernetic
 
 After cloning the repository
 ```bash
-cargo build
+cargo build --release # this is needed to skip validation layers
 ```
 
 ## Usage

codegen/src/lib.rs

@@ -25,10 +25,6 @@ pub fn codegen_field_protocol(_input: proc_macro::TokenStream) -> proc_macro::To
 	codegen_protocol(FIELD_PROTOCOL)
 }
 #[proc_macro]
-pub fn codegen_data_protocol(_input: proc_macro::TokenStream) -> proc_macro::TokenStream {
-	codegen_protocol(DATA_PROTOCOL)
-}
-#[proc_macro]
 pub fn codegen_audio_protocol(_input: proc_macro::TokenStream) -> proc_macro::TokenStream {
 	codegen_protocol(AUDIO_PROTOCOL)
 }
@@ -64,11 +60,28 @@ fn codegen_protocol(protocol: &'static str) -> proc_macro::TokenStream {
 			};
 			let aspect = generate_aspect(&Aspect {
 				name: "interface".to_string(),
+				id: 0,
 				description: protocol.description.clone(),
 				inherits: vec![],
 				members: p.members,
 			});
-			quote!(#node_id #aspect)
+			quote! {
+				#node_id
+				#aspect
+				pub struct Interface;
+				impl crate::nodes::AspectIdentifier for Interface {
+					impl_aspect_for_interface_aspect_id!{}
+				}
+				impl crate::nodes::Aspect for Interface {
+					impl_aspect_for_interface_aspect!{}
+				}
+				pub fn create_interface(client: &std::sync::Arc<crate::core::client::Client>) -> crate::core::error::Result<()>{
+					let node = crate::nodes::Node::from_id(client,INTERFACE_NODE_ID,false);
+					node.add_aspect(Interface);
+					node.add_to_scenegraph()?;
+					Ok(())
+				}
+			}
 		})
 		.unwrap_or_default();
 	let custom_enums = protocol
@@ -130,7 +143,7 @@ fn generate_custom_union(custom_union: &CustomUnion) -> TokenStream {
 	quote! {
 		#[doc = #description]
 		#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)]
-		#[serde(untagged)]
+		#[serde(tag = "t", content = "c")]
 		pub enum #name {#option_decls}
 	}
 }
@@ -177,11 +190,12 @@ fn generate_aspect(aspect: &Aspect) -> TokenStream {
 		Span::call_site(),
 	);
 	let client_side_members = client_members
-		.map(generate_member)
+		.map(|m| generate_member(aspect.id, &aspect.name.to_case(Case::Snake), m))
 		.reduce(fold_tokens)
 		.map(|t| {
 			// TODO: properly import all dependencies
 			quote! {
+				#[allow(clippy::all)]
 				pub mod #client_mod_name {
 					use super::*;
 					#t
@@ -190,11 +204,6 @@ fn generate_aspect(aspect: &Aspect) -> TokenStream {
 		})
 		.unwrap_or_default();
 
-	let aspect_trait_name = Ident::new(
-		&format!("{}Aspect", &aspect.name.to_case(Case::Pascal)),
-		Span::call_site(),
-	);
-
 	let opcodes = aspect
 		.members
 		.iter()
@@ -219,31 +228,95 @@ fn generate_aspect(aspect: &Aspect) -> TokenStream {
 	let alias_info = generate_alias_info(aspect);
 
 	let server_side_members = server_members
-		.map(generate_member)
+		.map(|m| generate_member(aspect.id, &aspect.name.to_case(Case::Pascal), m))
 		.reduce(fold_tokens)
 		.unwrap_or_default();
-	let add_node_members = aspect
+	let aspect_trait_name = Ident::new(
+		&format!("{}Aspect", &aspect.name.to_case(Case::Pascal)),
+		Span::call_site(),
+	);
+	let run_signals = aspect
 		.members
 		.iter()
 		.filter(|m| m.side == Side::Server)
-		.map(generate_handler)
+		.filter(|m| m._type == MemberType::Signal)
+		.map(|m| generate_run_member(&aspect_trait_name, MemberType::Signal, m))
+		.reduce(fold_tokens)
+		.unwrap_or_default();
+	let run_methods = aspect
+		.members
+		.iter()
+		.filter(|m| m.side == Side::Server)
+		.filter(|m| m._type == MemberType::Method)
+		.map(|m| generate_run_member(&aspect_trait_name, MemberType::Method, m))
 		.reduce(fold_tokens)
-		.map(|members| {
-			quote! {
-				fn add_node_members(node: &crate::nodes::Node) {
-					#members
-				}
-			}
-		})
 		.unwrap_or_default();
 	let server_side_members = quote! {
+		#[allow(clippy::all)]
 		#[doc = #description]
 		pub trait #aspect_trait_name {
-			#add_node_members
 			#server_side_members
 		}
 	};
-	quote!(#opcodes #alias_info #client_side_members #server_side_members)
+	let aspect_id_macro_name = Ident::new(
+		&format!(
+			"impl_aspect_for_{}_aspect_id",
+			aspect.name.to_case(Case::Snake)
+		),
+		Span::call_site(),
+	);
+	let aspect_macro_name = Ident::new(
+		&format!(
+			"impl_aspect_for_{}_aspect",
+			aspect.name.to_case(Case::Snake)
+		),
+		Span::call_site(),
+	);
+	let aspect_id = aspect.id;
+	let aspect_macro = quote! {
+		macro_rules! #aspect_id_macro_name {
+			() => {
+				const ID: u64 = #aspect_id;
+			}
+		}
+		macro_rules! #aspect_macro_name {
+			() => {
+				fn as_any(self: Arc<Self>) -> Arc<dyn std::any::Any + Send + Sync + 'static> {
+					self
+				}
+				#[allow(clippy::all)]
+				fn run_signal(
+					&self,
+					_calling_client: std::sync::Arc<crate::core::client::Client>,
+					_node: std::sync::Arc<crate::nodes::Node>,
+					_signal: u64,
+					_message: crate::nodes::Message
+				) -> Result<(), stardust_xr::scenegraph::ScenegraphError> {
+					match _signal {
+						#run_signals
+						_ => Err(stardust_xr::scenegraph::ScenegraphError::MemberNotFound)
+					}
+				}
+				#[allow(clippy::all)]
+				fn run_method(
+					&self,
+					_calling_client: std::sync::Arc<crate::core::client::Client>,
+					_node: std::sync::Arc<crate::nodes::Node>,
+					_method: u64,
+					_message: crate::nodes::Message,
+					_method_response: crate::nodes::MethodResponseSender,
+				) {
+					match _method {
+						#run_methods
+						_ => {
+							let _ = _method_response.send(Err(stardust_xr::scenegraph::ScenegraphError::MemberNotFound));
+						}
+					}
+				}
+			};
+		}
+	};
+	quote!(#opcodes #alias_info #client_side_members #server_side_members #aspect_macro)
 }
 
 fn generate_alias_opcodes(aspect: &Aspect, side: Side, _type: MemberType) -> TokenStream {
@@ -283,6 +356,7 @@ fn generate_alias_info(aspect: &Aspect) -> TokenStream {
 
 	quote! {
 		lazy_static::lazy_static! {
+			#[allow(clippy::all)]
 			pub static ref #aspect_alias_info_name: crate::nodes::alias::AliasInfo = crate::nodes::alias::AliasInfo {
 				server_signals: vec![#local_signals],
 				server_methods: vec![#local_methods],
@@ -293,8 +367,8 @@ fn generate_alias_info(aspect: &Aspect) -> TokenStream {
 	}
 }
 
-fn generate_member(member: &Member) -> TokenStream {
-	let id = member.opcode;
+fn generate_member(aspect_id: u64, aspect_name: &str, member: &Member) -> TokenStream {
+	let opcode = member.opcode;
 	let name = Ident::new(&member.name.to_case(Case::Snake), Span::call_site());
 	let description = &member.description;
 
@@ -324,40 +398,53 @@ fn generate_member(member: &Member) -> TokenStream {
 		.as_ref()
 		.map(|r| generate_argument_type(r, false, true))
 		.unwrap_or_else(|| quote!(()));
+	let name_str = name.to_string();
 
 	match (side, _type) {
-		(Side::Client, MemberType::Method) => {
-			quote! {
-				#[doc = #description]
-				pub async fn #name(#argument_decls) -> color_eyre::eyre::Result<(#return_type, Vec<std::os::fd::OwnedFd>)> {
-					_node.execute_remote_method_typed(#id, &(#argument_uses), vec![]).await
-				}
-			}
-		}
 		(Side::Client, MemberType::Signal) => {
 			quote! {
 				#[doc = #description]
-				pub fn #name(#argument_decls) -> color_eyre::eyre::Result<()> {
-					let serialized = stardust_xr::schemas::flex::serialize((#argument_uses))?;
-					_node.send_remote_signal(#id, serialized)
+				pub fn #name(#argument_decls) -> crate::core::error::Result<()> {
+
+					let result = stardust_xr::schemas::flex::serialize((#argument_uses)).map_err(|e|e.into()).and_then(|serialized|_node.send_remote_signal(#aspect_id, #opcode, serialized));
+					if let Err(err) = result.as_ref() {
+						::tracing::warn!("failed to send remote signal: {}::{}, error: {}",#aspect_name,#name_str,err);
+					} else {
+						::tracing::trace!("sent remote signal: {}::{}",#aspect_name,#name_str);
+					}
+					result
 				}
 			}
 		}
-		(Side::Server, MemberType::Method) => {
+		(Side::Client, MemberType::Method) => {
 			quote! {
 				#[doc = #description]
-				fn #name(#argument_decls) -> impl std::future::Future<Output = color_eyre::eyre::Result<#return_type>> + Send + 'static;
+				pub async fn #name(#argument_decls) -> crate::core::error::Result<(#return_type, Vec<std::os::fd::OwnedFd>)> {
+					let result = _node.execute_remote_method_typed(#aspect_id, #opcode, &(#argument_uses), vec![]).await;
+					if let Err(err) = result.as_ref() {
+						::tracing::warn!("failed to call remote method: {}::{}, error: {}",#aspect_name,#name_str,err);
+					} else {
+						::tracing::trace!("called remote method: {}::{}",#aspect_name,#name_str);
+					}
+					result
+				}
 			}
 		}
 		(Side::Server, MemberType::Signal) => {
 			quote! {
 				#[doc = #description]
-				fn #name(#argument_decls) -> color_eyre::eyre::Result<()>;
+				fn #name(#argument_decls) -> crate::core::error::Result<()>;
+			}
+		}
+		(Side::Server, MemberType::Method) => {
+			quote! {
+				#[doc = #description]
+				fn #name(#argument_decls) -> impl std::future::Future<Output = crate::core::error::Result<#return_type>> + Send + Sync + 'static;
 			}
 		}
 	}
 }
-fn generate_handler(member: &Member) -> TokenStream {
+fn generate_run_member(aspect_name: &Ident, _type: MemberType, member: &Member) -> TokenStream {
 	let opcode = member.opcode;
 	let member_name_ident = Ident::new(&member.name, Span::call_site());
 
@@ -379,7 +466,10 @@ fn generate_handler(member: &Member) -> TokenStream {
 		.clone()
 		.zip(argument_types)
 		.map(|(argument_names, argument_types)| {
-			quote!(let (#argument_names): (#argument_types) = stardust_xr::schemas::flex::deserialize(_message.as_ref())?;)
+			quote!{
+				#[allow(unused_parens)]
+				let (#argument_names): (#argument_types) = stardust_xr::schemas::flex::deserialize(_message.as_ref())?;
+			}
 		})
 		.unwrap_or_default();
 	let serialize = generate_argument_serialize(
@@ -393,21 +483,34 @@ fn generate_handler(member: &Member) -> TokenStream {
 		.map(|a| generate_argument_deserialize(&a.name, &a._type, a.optional))
 		.reduce(|a, b| quote!(#a, #b))
 		.unwrap_or_default();
-	match member._type {
+	let member_name = member_name_ident.to_string();
+	let aspect_name_str = aspect_name.to_string();
+	match _type {
 		MemberType::Signal => quote! {
-			node.add_local_signal(#opcode, |_node, _calling_client, _message| {
+			#opcode => { let result = (move || {
 				#deserialize
-				Self::#member_name_ident(_node, _calling_client.clone(), #argument_uses)
-			});
+				<Self as #aspect_name>::#member_name_ident(_node, _calling_client.clone(), #argument_uses)
+			})().map_err(|e: crate::core::error::ServerError| stardust_xr::scenegraph::ScenegraphError::MemberError { error: e.to_string() });
+				if let Err(err) = result.as_ref() {
+					::tracing::warn!("failed to receive local signal: {}::{}, error: {}",#aspect_name_str,#member_name,err);
+				} else {
+					::tracing::trace!("received local signal: {}::{}",#aspect_name_str,#member_name);
+				}
+				result
+			},
 		},
 		MemberType::Method => quote! {
-			node.add_local_method(#opcode, |_node, _calling_client, _message, _method_response| {
-				_method_response.wrap_async(async move {
+			#opcode => _method_response.wrap_async(async move {
 					#deserialize
-					let result = Self::#member_name_ident(_node, _calling_client.clone(), #argument_uses).await?;
-					Ok((#serialize, Vec::new()))
-				});
-			});
+					let result = <Self as #aspect_name>::#member_name_ident(_node, _calling_client.clone(), #argument_uses).await;
+					if let Err(err) = result.as_ref() {
+						::tracing::warn!("failed to call local method: {}::{}, error: {}",#aspect_name_str,#member_name,err);
+					} else {
+						::tracing::trace!("called local method: {}::{}",#aspect_name_str,#member_name);
+					};
+					let result = result?;
+					 Ok((#serialize, Vec::<std::os::fd::OwnedFd>::new()))
+			}),
 		},
 	}
 }
@@ -441,18 +544,18 @@ fn generate_argument_deserialize(
 	}
 	if optional {
 		let mapping = generate_argument_deserialize("o", argument_type, false);
-		return quote!(#name.map(|o| Ok::<_, color_eyre::eyre::Report>(#mapping)).transpose()?);
+		return quote!(#name.map(|o| Ok::<_, crate::core::error::ServerError>(#mapping)).transpose()?);
 	}
 
 	match argument_type {
 		ArgumentType::Color => quote!(color::rgba_linear!(#name[0], #name[1], #name[2], #name[3])),
 		ArgumentType::Vec(v) => {
 			let mapping = generate_argument_deserialize("a", v, false);
-			quote!(#name.into_iter().map(|a| Ok(#mapping)).collect::<color_eyre::eyre::Result<Vec<_>>>()?)
+			quote!(#name.into_iter().map(|a| Ok(#mapping)).collect::<crate::core::error::Result<Vec<_>>>()?)
 		}
 		ArgumentType::Map(v) => {
 			let mapping = generate_argument_deserialize("a", v, false);
-			quote!(#name.into_iter().map(|(k, a)| Ok((k, #mapping))).collect::<color_eyre::eyre::Result<rustc_hash::FxHashMap<String, _>>>()?)
+			quote!(#name.into_iter().map(|(k, a)| Ok((k, #mapping))).collect::<crate::core::error::Result<rustc_hash::FxHashMap<String, _>>>()?)
 		}
 		_ => quote!(#name),
 	}
@@ -474,11 +577,11 @@ fn generate_argument_serialize(
 		ArgumentType::Color => quote!([#name.c.r, #name.c.g, #name.c.b, #name.a]),
 		ArgumentType::Vec(v) => {
 			let mapping = generate_argument_serialize("a", v, false);
-			quote!(#name.into_iter().map(|a| Ok(#mapping)).collect::<color_eyre::eyre::Result<Vec<_>>>()?)
+			quote!(#name.into_iter().map(|a| Ok(#mapping)).collect::<crate::core::error::Result<Vec<_>>>()?)
 		}
 		ArgumentType::Map(v) => {
 			let mapping = generate_argument_serialize("a", v, false);
-			quote!(#name.into_iter().map(|(k, a)| Ok((k, #mapping))).collect::<color_eyre::eyre::Result<rustc_hash::FxHashMap<String, _>>>()?)
+			quote!(#name.into_iter().map(|(k, a)| Ok((k, #mapping))).collect::<crate::core::error::Result<rustc_hash::FxHashMap<String, _>>>()?)
 		}
 		_ => quote!(#name),
 	}
@@ -511,6 +614,7 @@ fn argument_type_option_name(argument_type: &ArgumentType) -> String {
 		ArgumentType::Union(u) => u.clone(),
 		ArgumentType::Struct(s) => s.clone(),
 		ArgumentType::Node { _type, .. } => _type.clone(),
+		ArgumentType::Fd => "File Descriptor".to_string(),
 	}
 }
 fn generate_argument_type(
@@ -607,6 +711,9 @@ fn generate_argument_type(
 				quote!(std::sync::Arc<crate::nodes::Node>)
 			}
 		}
+		ArgumentType::Fd => {
+			quote!(&std::os::fd::OwnedFd)
+		}
 	};
 
 	if optional {

flake.nix

@@ -15,13 +15,21 @@
     flatland.url = "github:StardustXR/flatland";
   };
   outputs =
-    inputs@{ self, flake-parts, nixpkgs, hercules-ci-effects, flatland, ... }:
+    inputs@{
+      self,
+      flake-parts,
+      nixpkgs,
+      hercules-ci-effects,
+      flatland,
+      ...
+    }:
     let
       name = (builtins.fromTOML (builtins.readFile ./Cargo.toml)).package.name;
       src = builtins.path {
         name = "${name}-source";
         path = toString ./.;
-        filter = path: type:
+        filter =
+          path: type:
           nixpkgs.lib.all (n: builtins.baseNameOf path != n) [
             "flake.nix"
             "flake.lock"
@@ -29,17 +37,34 @@
             "README.md"
           ];
       };
-    in flake-parts.lib.mkFlake { inherit inputs; } {
+    in
+    flake-parts.lib.mkFlake { inherit inputs; } {
       imports = [ flake-parts.flakeModules.easyOverlay ];
-      systems = [ "aarch64-linux" "x86_64-linux" "riscv64-linux" ];
-      perSystem = { config, self', inputs', pkgs, system, ... }: {
+      systems = [
+        "aarch64-linux"
+        "x86_64-linux"
+        "riscv64-linux"
+      ];
+      perSystem =
+        {
+          config,
+          self',
+          inputs',
+          pkgs,
+          system,
+          ...
+        }:
+        {
           _module.args.pkgs = import inputs.nixpkgs {
             inherit system;
             overlays = [ inputs.self.overlays.default ];
           };
           overlayAttrs = config.packages;
-        packages = let sk_gpu = pkgs.callPackage ./nix/sk_gpu.nix { };
-        in {
+          packages =
+            let
+              sk_gpu = pkgs.callPackage ./nix/sk_gpu.nix { };
+            in
+            {
               default = self'.packages.${name};
               gnome-graphical-test = self'.checks.gnome-graphical-test;
               "${name}" = pkgs.callPackage ./nix/stardust-xr-server.nix {
@@ -50,8 +75,9 @@
             type = "app";
             program = self'.packages.${name} + "/bin/stardust-xr-server";
           };
-        checks.gnome-graphical-test = pkgs.nixosTest
-          (import ./nix/gnome-graphical-test.nix { inherit pkgs self; });
+          checks.gnome-graphical-test = pkgs.nixosTest (
+            import ./nix/gnome-graphical-test.nix { inherit pkgs self; }
+          );
           devShells.default = pkgs.mkShell {
             inputsFrom = [ self'.packages.default ];
             LIBCLANG_PATH = "${pkgs.libclang.lib}/lib";
@@ -59,10 +85,13 @@
         };
       flake = {
         herculesCI.ciSystems = [ "x86_64-linux" ];
-        effects = let
+        effects =
+          let
             pkgs = nixpkgs.legacyPackages.x86_64-linux;
             hci-effects = hercules-ci-effects.lib.withPkgs pkgs;
-        in { ref, rev, ... }: {
+          in
+          { ref, rev, ... }:
+          {
             gnome-graphical-test = hci-effects.mkEffect {
               secretsMap."stardustxrDiscord" = "stardustxrDiscord";
               secretsMap."stardustxrIpfs" = "stardustxrIpfs";

justfile

@@ -0,0 +1,12 @@
+PREFIX := "usr"
+BINARY := PREFIX / "bin"
+DESTDIR := "/"
+
+build:
+    cargo build --release
+
+test:
+	cargo test
+
+install:
+    install -Dm755 target/release/stardust-xr-server {{ DESTDIR }}{{ BINARY }}/stardust-xr-server

src/core/bevy_channel.rs

@@ -0,0 +1,31 @@
+use std::sync::OnceLock;
+
+use bevy::prelude::*;
+use tokio::sync::mpsc::{self, error::TryRecvError};
+
+#[derive(Resource)]
+pub struct BevyChannelReader<T: Send + Sync + 'static>(mpsc::UnboundedReceiver<T>);
+pub struct BevyChannel<T: Send + Sync + 'static>(OnceLock<mpsc::UnboundedSender<T>>);
+impl<T: Send + Sync + 'static> BevyChannel<T> {
+	pub const fn new() -> Self {
+		Self(OnceLock::new())
+	}
+	pub fn init(&self, app: &mut App) {
+		let (tx, rx) = mpsc::unbounded_channel();
+		self.0.set(tx).unwrap();
+		app.insert_resource(BevyChannelReader(rx));
+	}
+	pub fn send(&self, msg: T) -> Option<()> {
+		self.0.get()?.send(msg).ok()
+	}
+}
+
+impl<T: Send + Sync + 'static> BevyChannelReader<T> {
+	pub fn read(&mut self) -> Option<T> {
+		match self.0.try_recv() {
+			Ok(v) => Some(v),
+			Err(TryRecvError::Disconnected) => panic!("bevy channel should never disconnect"),
+			Err(TryRecvError::Empty) => None,
+		}
+	}
+}

src/core/client.rs

@@ -1,46 +1,57 @@
 use super::{
-	client_state::{ClientStateParsed, CLIENT_STATES},
-	destroy_queue,
+	client_state::{CLIENT_STATES, ClientStateParsed},
 	scenegraph::Scenegraph,
 };
 use crate::{
 	core::{registry::OwnedRegistry, task},
 	nodes::{
-		audio, data, drawable, fields, input, items,
+		Node, audio, drawable, fields, input, items,
 		root::{ClientState, Root},
-		spatial, Node,
+		spatial,
 	},
 };
-use color_eyre::eyre::{eyre, Result};
+use color_eyre::eyre::{Result, eyre};
 use global_counter::primitive::exact::CounterU32;
 use lazy_static::lazy_static;
-use once_cell::sync::OnceCell;
 use parking_lot::Mutex;
 use rustc_hash::FxHashMap;
 use stardust_xr::messenger::{self, MessageSenderHandle};
-use std::{fmt::Debug, fs, iter::FromIterator, path::PathBuf, sync::Arc};
-use tokio::{net::UnixStream, task::JoinHandle};
+use std::{
+	fmt::Debug,
+	fs,
+	iter::FromIterator,
+	path::PathBuf,
+	sync::{Arc, OnceLock},
+	time::Instant,
+};
+use tokio::{net::UnixStream, sync::watch, task::JoinHandle};
 use tracing::info;
 
+pub static CLIENTS: OwnedRegistry<Client> = OwnedRegistry::new();
+
 lazy_static! {
-	pub static ref CLIENTS: OwnedRegistry<Client> = OwnedRegistry::new();
+	static ref INTERNAL_CLIENT_MESSAGE_TIMES: (watch::Sender<Instant>, watch::Receiver<Instant>) = watch::channel(Instant::now());
 	pub static ref INTERNAL_CLIENT: Arc<Client> = CLIENTS.add(Client {
 		pid: None,
 		// env: None,
 		exe: None,
 
-		dispatch_join_handle: OnceCell::new(),
-		flush_join_handle: OnceCell::new(),
-		disconnect_status: OnceCell::new(),
+		dispatch_join_handle: OnceLock::new(),
+		flush_join_handle: OnceLock::new(),
+		disconnect_status: OnceLock::new(),
 
-		message_sender_handle: None,
 		id_counter: CounterU32::new(0),
+		message_last_received: INTERNAL_CLIENT_MESSAGE_TIMES.1.clone(),
+		message_sender_handle: None,
 		scenegraph: Default::default(),
-		root: OnceCell::new(),
+		root: OnceLock::new(),
 		base_resource_prefixes: Default::default(),
-		state: OnceCell::default(),
+		state: OnceLock::default(),
 	});
 }
+pub fn tick_internal_client() {
+	let _ = INTERNAL_CLIENT_MESSAGE_TIMES.0.send(Instant::now());
+}
 
 pub fn get_env(pid: i32) -> Result<FxHashMap<String, String>, std::io::Error> {
 	let env = fs::read_to_string(format!("/proc/{pid}/environ"))?;
@@ -59,22 +70,23 @@ pub struct Client {
 	pub pid: Option<i32>,
 	// env: Option<FxHashMap<String, String>>,
 	exe: Option<PathBuf>,
-	dispatch_join_handle: OnceCell<JoinHandle<Result<()>>>,
-	flush_join_handle: OnceCell<JoinHandle<Result<()>>>,
-	disconnect_status: OnceCell<Result<()>>,
+	dispatch_join_handle: OnceLock<JoinHandle<Result<()>>>,
+	flush_join_handle: OnceLock<JoinHandle<Result<()>>>,
+	disconnect_status: OnceLock<Result<()>>,
 
 	id_counter: CounterU32,
+	message_last_received: watch::Receiver<Instant>,
 	pub message_sender_handle: Option<MessageSenderHandle>,
 	pub scenegraph: Arc<Scenegraph>,
-	pub root: OnceCell<Arc<Root>>,
+	pub root: OnceLock<Arc<Root>>,
 	pub base_resource_prefixes: Mutex<Vec<PathBuf>>,
-	pub state: OnceCell<ClientState>,
+	pub state: OnceLock<ClientState>,
 }
 impl Client {
 	pub fn from_connection(connection: UnixStream) -> Result<Arc<Self>> {
 		let pid = connection.peer_cred().ok().and_then(|c| c.pid());
 		let env = pid.and_then(|pid| get_env(pid).ok());
-		let exe = pid.and_then(|pid| fs::read_link(format!("/proc/{}/exe", pid)).ok());
+		let exe = pid.and_then(|pid| fs::read_link(format!("/proc/{pid}/exe")).ok());
 		info!(
 			pid,
 			exe = exe
@@ -90,21 +102,23 @@ impl Client {
 			.and_then(state)
 			.unwrap_or_else(|| Arc::new(ClientStateParsed::default()));
 
+		let (message_time_tx, message_last_received) = watch::channel(Instant::now());
 		let client = CLIENTS.add(Client {
 			pid,
 			// env,
 			exe: exe.clone(),
 
-			dispatch_join_handle: OnceCell::new(),
-			flush_join_handle: OnceCell::new(),
-			disconnect_status: OnceCell::new(),
+			dispatch_join_handle: OnceLock::new(),
+			flush_join_handle: OnceLock::new(),
+			disconnect_status: OnceLock::new(),
 
 			id_counter: CounterU32::new(256),
+			message_last_received,
 			message_sender_handle: Some(messenger_tx.handle()),
 			scenegraph: scenegraph.clone(),
-			root: OnceCell::new(),
+			root: OnceLock::new(),
 			base_resource_prefixes: Default::default(),
-			state: OnceCell::default(),
+			state: OnceLock::default(),
 		});
 		let _ = client.scenegraph.client.set(Arc::downgrade(&client));
 		let _ = client.root.set(Root::create(&client, state.root)?);
@@ -112,7 +126,6 @@ impl Client {
 		fields::create_interface(&client)?;
 		drawable::create_interface(&client)?;
 		audio::create_interface(&client)?;
-		data::create_interface(&client)?;
 		input::create_interface(&client)?;
 		items::camera::create_interface(&client)?;
 		items::panel::create_interface(&client)?;
@@ -129,7 +142,7 @@ impl Client {
 					.map(|exe| exe.to_string())
 			})
 			.unwrap_or_else(|| "??".to_string());
-		let _ = client.dispatch_join_handle.get_or_try_init(|| {
+		let _ = client.dispatch_join_handle.get_or_init(|| {
 			task::new(
 				|| {
 					format!(
@@ -144,12 +157,14 @@ impl Client {
 							if let Err(e) = messenger_rx.dispatch(&*scenegraph).await {
 								client.disconnect(Err(e.into()));
 							}
+							let _ = message_time_tx.send(Instant::now());
 						}
 					}
 				},
 			)
+			.unwrap()
 		});
-		let _ = client.flush_join_handle.get_or_try_init(|| {
+		let _ = client.flush_join_handle.get_or_init(|| {
 			task::new(
 				|| format!("client flush pid={} exe={}", &pid_printable, &exe_printable,),
 				{
@@ -163,6 +178,7 @@ impl Client {
 					}
 				},
 			)
+			.unwrap()
 		});
 
 		Ok(client)
@@ -185,7 +201,9 @@ impl Client {
 		std::fs::read_link(cwd_proc_path).ok()
 	}
 	pub async fn save_state(&self) -> Option<ClientStateParsed> {
+		println!("start save state");
 		let internal = self.root.get()?.save_state().await.ok()?;
+		println!("finished save state");
 		Some(ClientStateParsed::from_deserialized(self, internal))
 	}
 
@@ -200,6 +218,11 @@ impl Client {
 			.ok_or_else(|| eyre!("{} not found", name))
 	}
 
+	pub fn unresponsive(&self) -> bool {
+		let time_since_last_message = self.message_last_received.borrow().elapsed();
+		time_since_last_message.as_millis() > 500
+	}
+
 	pub fn disconnect(&self, reason: Result<()>) {
 		let _ = self.disconnect_status.set(reason);
 		if let Some(dispatch_join_handle) = self.dispatch_join_handle.get() {
@@ -208,9 +231,7 @@ impl Client {
 		if let Some(flush_join_handle) = self.flush_join_handle.get() {
 			flush_join_handle.abort();
 		}
-		if let Some(client) = CLIENTS.remove(self) {
-			destroy_queue::add(client);
-		}
+		CLIENTS.remove(self);
 	}
 }
 impl Debug for Client {

src/core/client_state.rs

@@ -1,5 +1,5 @@
-use super::client::{get_env, Client};
-use crate::nodes::{root::ClientState, spatial::Spatial, Node};
+use super::client::{Client, get_env};
+use crate::nodes::{Node, root::ClientState, spatial::Spatial};
 use glam::Mat4;
 use parking_lot::Mutex;
 use rustc_hash::FxHashMap;
@@ -69,7 +69,7 @@ impl ClientStateParsed {
 		let app_name = self
 			.launch_info
 			.as_ref()
-			.map(|l| l.cmdline.first().unwrap().split('/').last().unwrap())
+			.map(|l| l.cmdline.first().unwrap().split('/').next_back().unwrap())
 			.unwrap_or("unknown");
 		let state_file_path = directory
 			.join(format!("{app_name}-{}", nanoid::nanoid!()))

src/core/color.rs

@@ -0,0 +1,12 @@
+use stardust_xr::values::color::{AlphaColor, Rgb, color_space::LinearRgb};
+
+pub trait ColorConvert {
+	fn to_bevy(&self) -> bevy::color::Color;
+}
+// even tho its supposed to be linear the values have to be interpreted as Srgba to produce the
+// correct result because StereoKit used Srgba while it was assumed that is uses linear rgba
+impl ColorConvert for AlphaColor<f32, Rgb<f32, LinearRgb>> {
+	fn to_bevy(&self) -> bevy::color::Color {
+		bevy::color::Color::srgba(self.c.r, self.c.g, self.c.b, self.a)
+	}
+}

src/core/destroy_queue.rs

@@ -1,24 +0,0 @@
-use once_cell::sync::Lazy;
-use parking_lot::Mutex;
-use std::any::Any;
-use tokio::sync::mpsc::{self, unbounded_channel};
-
-type Anything = Box<dyn Any + Send + Sync>;
-
-static MAIN_DESTROY_QUEUE: Lazy<(
-	mpsc::UnboundedSender<Anything>,
-	Mutex<mpsc::UnboundedReceiver<Anything>>,
-)> = Lazy::new(|| {
-	let (tx, rx) = unbounded_channel();
-	(tx, Mutex::new(rx))
-});
-
-pub fn add<T: Any + Sync + Send>(thing: T) {
-	MAIN_DESTROY_QUEUE.0.send(Box::new(thing)).unwrap();
-}
-
-pub fn clear() {
-	while let Ok(thing) = MAIN_DESTROY_QUEUE.1.lock().try_recv() {
-		drop(thing)
-	}
-}

src/core/entity_handle.rs

@@ -0,0 +1,33 @@
+use bevy::prelude::*;
+
+use super::bevy_channel::{BevyChannel, BevyChannelReader};
+pub struct EntityHandlePlugin;
+
+impl Plugin for EntityHandlePlugin {
+	fn build(&self, app: &mut App) {
+		DESTROY.init(app);
+		app.add_systems(PreUpdate, despawn);
+	}
+}
+
+fn despawn(mut cmds: Commands, mut reader: ResMut<BevyChannelReader<Entity>>) {
+	while let Some(e) = reader.read() {
+		cmds.entity(e).try_despawn();
+	}
+}
+
+static DESTROY: BevyChannel<Entity> = BevyChannel::new();
+#[derive(Deref, DerefMut, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)]
+pub struct EntityHandle(pub Entity);
+impl Drop for EntityHandle {
+	fn drop(&mut self) {
+		if DESTROY.send(self.0).is_none() {
+			error!("Entity Destroy channel not open");
+		}
+	}
+}
+impl From<Entity> for EntityHandle {
+	fn from(value: Entity) -> Self {
+		Self(value)
+	}
+}

src/core/error.rs

@@ -0,0 +1,75 @@
+use std::any::TypeId;
+
+use color_eyre::eyre::Report;
+use stardust_xr::{
+	messenger::MessengerError,
+	schemas::flex::{
+		FlexSerializeError,
+		flexbuffers::{DeserializationError, ReaderError},
+	},
+};
+use thiserror::Error;
+
+pub type Result<T, E = ServerError> = std::result::Result<T, E>;
+
+#[derive(Error, Debug)]
+pub enum ServerError {
+	#[error("Internal: Unable to get client")]
+	NoClient,
+	#[error("Messenger does not exist for this node")]
+	NoMessenger,
+	#[error("Messenger error: {0}")]
+	MessengerError(#[from] MessengerError),
+	#[error("Remote method error: {0}")]
+	RemoteMethodError(String),
+	#[error("Serialization error: {0}")]
+	SerializationError(#[from] FlexSerializeError),
+	#[error("Deserialization error: {0}")]
+	DeserializationError(#[from] DeserializationError),
+	#[error("Reader error: {0}")]
+	ReaderError(#[from] ReaderError),
+	#[cfg(feature = "wayland")]
+	#[error("Wayland error: {0}")]
+	WaylandError(waynest::server::Error),
+	#[error("Aspect {} does not exist for node", 0.to_string())]
+	NoAspect(TypeId),
+	#[error("{0}")]
+	Report(#[from] Report),
+}
+
+#[macro_export]
+macro_rules! bail {
+    ($msg:literal $(,)?) => {
+        return Err($crate::core::error::ServerError::from(color_eyre::eyre::eyre!($msg)));
+    };
+    ($err:expr $(,)?) => {
+        return Err($crate::core::error::ServerError::from(color_eyre::eyre::eyre!($err)));
+    };
+    ($fmt:expr, $($arg:tt)*) => {
+        return Err($crate::core::error::ServerError::from(color_eyre::eyre::eyre!($fmt, $($arg)*)));
+    };
+}
+
+#[macro_export]
+macro_rules! ensure {
+    ($cond:expr $(,)?) => {
+        if !$cond {
+            $crate::ensure!($cond, concat!("Condition failed: `", stringify!($cond), "`"))
+        }
+    };
+    ($cond:expr, $msg:literal $(,)?) => {
+        if !$cond {
+            return Err($crate::core::error::ServerError::from(color_eyre::eyre::eyre!($msg)));
+        }
+    };
+    ($cond:expr, $err:expr $(,)?) => {
+        if !$cond {
+            return Err($crate::core::error::ServerError::from(color_eyre::eyre::eyre!($err)));
+        }
+    };
+    ($cond:expr, $fmt:expr, $($arg:tt)*) => {
+        if !$cond {
+            return Err($crate::core::error::ServerError::from(color_eyre::eyre::eyre!($fmt, $($arg)*)));
+        }
+    };
+}

src/core/idl_utils.rs

@@ -1,11 +0,0 @@
-#[macro_export]
-macro_rules! create_interface {
-	($iface:ident) => {
-		pub fn create_interface(client: &Arc<Client>) -> Result<()> {
-			let node = Node::from_id(client, INTERFACE_NODE_ID, false);
-			<$iface as self::InterfaceAspect>::add_node_members(&node);
-			node.add_to_scenegraph()?;
-			Ok(())
-		}
-	};
-}

src/core/mod.rs

@@ -1,8 +1,10 @@
+pub mod bevy_channel;
 pub mod client;
 pub mod client_state;
+pub mod color;
 pub mod delta;
-pub mod destroy_queue;
-pub mod idl_utils;
+pub mod entity_handle;
+pub mod error;
 pub mod registry;
 pub mod resource;
 pub mod scenegraph;

src/core/registry.rs

@@ -1,19 +1,18 @@
 #![allow(dead_code)]
 
-use parking_lot::{const_mutex, MappedMutexGuard, Mutex, MutexGuard};
+use dashmap::DashMap;
+use parking_lot::{MappedMutexGuard, Mutex, MutexGuard, const_mutex};
 use rustc_hash::FxHashMap;
+use std::ops::Deref;
 use std::ptr;
-use std::sync::{Arc, Weak};
+use std::sync::{Arc, LazyLock, Weak};
 
 #[derive(Debug)]
-pub struct Registry<T: Send + Sync + ?Sized>(Mutex<Option<FxHashMap<usize, Weak<T>>>>);
+pub struct Registry<T: Send + Sync + ?Sized>(MaybeLazy<DashMap<usize, Weak<T>>>);
 
 impl<T: Send + Sync + ?Sized> Registry<T> {
 	pub const fn new() -> Self {
-		Registry(const_mutex(None))
-	}
-	fn lock(&self) -> MappedMutexGuard<FxHashMap<usize, Weak<T>>> {
-		MutexGuard::map(self.0.lock(), |r| r.get_or_insert_with(FxHashMap::default))
+		Registry(MaybeLazy::Lazy(LazyLock::new(DashMap::default)))
 	}
 	pub fn add(&self, t: T) -> Arc<T>
 	where
@@ -24,30 +23,29 @@ impl<T: Send + Sync + ?Sized> Registry<T> {
 		t_arc
 	}
 	pub fn add_raw(&self, t: &Arc<T>) {
-		self.lock()
+		self.0
 			.insert(Arc::as_ptr(t) as *const () as usize, Arc::downgrade(t));
 	}
 	pub fn contains(&self, t: &T) -> bool {
-		self.lock()
+		self.0
 			.contains_key(&(ptr::addr_of!(*t) as *const () as usize))
 	}
 	pub fn get_changes(old: &Registry<T>, new: &Registry<T>) -> (Vec<Arc<T>>, Vec<Arc<T>>) {
-		let old = old.lock();
-		let new = new.lock();
-
 		let mut added = Vec::new();
 		let mut removed = Vec::new();
 
-		for (id, entry) in new.iter() {
+		for pair in new.0.iter() {
+			let (id, entry) = pair.pair();
 			if let Some(entry) = entry.upgrade() {
-				if !old.contains_key(id) {
+				if !old.0.contains_key(id) {
 					added.push(entry);
 				}
 			}
 		}
-		for (id, entry) in old.iter() {
+		for pair in old.0.iter() {
+			let (id, entry) = pair.pair();
 			if let Some(entry) = entry.upgrade() {
-				if !new.contains_key(id) {
+				if !new.0.contains_key(id) {
 					removed.push(entry);
 				}
 			}
@@ -55,52 +53,48 @@ impl<T: Send + Sync + ?Sized> Registry<T> {
 		(added, removed)
 	}
 	pub fn get_valid_contents(&self) -> Vec<Arc<T>> {
-		self.lock()
+		self.0
 			.iter()
-			.filter_map(|pair| pair.1.upgrade())
+			.filter_map(|pair| pair.value().upgrade())
 			.collect()
 	}
 	pub fn set(&self, other: &Registry<T>) {
-		self.lock().clone_from(&other.lock());
+		self.clear();
+		for (key, value) in other.0.deref().clone().into_iter() {
+			self.0.insert(key, value);
+		}
 	}
 	pub fn take_valid_contents(&self) -> Vec<Arc<T>> {
-		self.0
-			.lock()
-			.take()
-			.unwrap_or_default()
-			.into_iter()
-			.filter_map(|pair| pair.1.upgrade())
-			.collect()
+		let contents = self.get_valid_contents();
+		self.0.clear();
+		contents
 	}
 	pub fn retain<F: Fn(&Arc<T>) -> bool>(&self, f: F) {
-		self.lock().retain(|_, v| {
+		self.0.retain(|_, v| {
 			let Some(v) = v.upgrade() else {
+				// why would we want to retain things we can't upgrade?
 				return true;
 			};
 			(f)(&v)
 		})
 	}
 	pub fn remove(&self, t: &T) {
-		self.lock()
-			.remove(&(ptr::addr_of!(*t) as *const () as usize));
+		self.0.remove(&(ptr::addr_of!(*t) as *const () as usize));
 	}
 	pub fn clear(&self) {
-		self.lock().clear();
+		self.0.clear();
 	}
 	pub fn is_empty(&self) -> bool {
-		let registry = self.0.lock();
-		let Some(registry) = &*registry else {
-			return true;
-		};
-		if registry.is_empty() {
+		if self.0.is_empty() {
 			return true;
 		}
-		registry.values().all(|v| v.strong_count() == 0)
+		self.0.iter().all(|v| v.value().strong_count() == 0)
 	}
 }
+
 impl<T: Send + Sync + ?Sized> Clone for Registry<T> {
 	fn clone(&self) -> Self {
-		Self(Mutex::new(self.0.lock().clone()))
+		Self(self.0.clone())
 	}
 }
 impl<T: Send + Sync + ?Sized> Default for Registry<T> {
@@ -109,6 +103,40 @@ impl<T: Send + Sync + ?Sized> Default for Registry<T> {
 	}
 }
 
+impl<T: Send + Sync + Sized> FromIterator<Arc<T>> for Registry<T> {
+	fn from_iter<I: IntoIterator<Item = Arc<T>>>(iter: I) -> Self {
+		Registry(MaybeLazy::NonLazy(
+			iter.into_iter()
+				.map(|i| (Arc::as_ptr(&i) as usize, Arc::downgrade(&i)))
+				.collect(),
+		))
+	}
+}
+
+#[derive(Debug)]
+enum MaybeLazy<T> {
+	Lazy(LazyLock<T>),
+	NonLazy(T),
+}
+impl<T: Clone> Clone for MaybeLazy<T> {
+	fn clone(&self) -> Self {
+		match self {
+			MaybeLazy::Lazy(lazy_lock) => Self::NonLazy(lazy_lock.deref().clone()),
+			MaybeLazy::NonLazy(v) => Self::NonLazy(v.clone()),
+		}
+	}
+}
+impl<T> Deref for MaybeLazy<T> {
+	type Target = T;
+
+	fn deref(&self) -> &Self::Target {
+		match self {
+			MaybeLazy::Lazy(lazy_lock) => lazy_lock,
+			MaybeLazy::NonLazy(v) => v,
+		}
+	}
+}
+
 pub struct OwnedRegistry<T: Send + Sync + ?Sized>(Mutex<Option<FxHashMap<usize, Arc<T>>>>);
 
 impl<T: Send + Sync + ?Sized> OwnedRegistry<T> {

src/core/scenegraph.rs

@@ -1,8 +1,7 @@
-use crate::nodes::alias::get_original;
+use crate::core::error::Result;
 use crate::nodes::Node;
+use crate::nodes::alias::get_original;
 use crate::{core::client::Client, nodes::Message};
-use color_eyre::eyre::Result;
-use once_cell::sync::OnceCell;
 use parking_lot::Mutex;
 use rustc_hash::FxHashMap;
 use serde::Serialize;
@@ -11,13 +10,13 @@ use stardust_xr::scenegraph::ScenegraphError;
 use stardust_xr::schemas::flex::serialize;
 use std::future::Future;
 use std::os::fd::OwnedFd;
-use std::sync::{Arc, Weak};
+use std::sync::{Arc, OnceLock, Weak};
 use tokio::sync::oneshot;
 use tracing::{debug, debug_span};
 
 #[derive(Default)]
 pub struct Scenegraph {
-	pub(super) client: OnceCell<Weak<Client>>,
+	pub(super) client: OnceLock<Weak<Client>>,
 	nodes: Mutex<FxHashMap<u64, Arc<Node>>>,
 }
 
@@ -59,26 +58,26 @@ impl MethodResponseSender {
 	// ) {
 	// 	let _ = self.0.send(map_method_return(result));
 	// }
-	pub fn wrap_sync<F: FnOnce() -> color_eyre::eyre::Result<Message>>(self, f: F) {
-		self.send(f().map_err(|e| ScenegraphError::MethodError {
+	pub fn wrap_sync<F: FnOnce() -> crate::core::error::Result<Message>>(self, f: F) {
+		self.send(f().map_err(|e| ScenegraphError::MemberError {
 			error: e.to_string(),
 		}))
 	}
 	pub fn wrap_async<T: Serialize>(
 		self,
-		f: impl Future<Output = color_eyre::eyre::Result<(T, Vec<OwnedFd>)>> + Send + 'static,
+		f: impl Future<Output = Result<(T, Vec<OwnedFd>)>> + Send + 'static,
 	) {
 		tokio::task::spawn(async move { self.0.send(map_method_return(f.await)) });
 	}
 }
 fn map_method_return<T: Serialize>(
-	result: color_eyre::eyre::Result<(T, Vec<OwnedFd>)>,
+	result: Result<(T, Vec<OwnedFd>)>,
 ) -> Result<(Vec<u8>, Vec<OwnedFd>), ScenegraphError> {
-	let (value, fds) = result.map_err(|e| ScenegraphError::MethodError {
+	let (value, fds) = result.map_err(|e| ScenegraphError::MemberError {
 		error: e.to_string(),
 	})?;
 
-	let serialized_value = serialize(value).map_err(|e| ScenegraphError::MethodError {
+	let serialized_value = serialize(value).map_err(|e| ScenegraphError::MemberError {
 		error: format!("Internal: Serialization failed: {e}"),
 	})?;
 	Ok((serialized_value, fds))
@@ -86,19 +85,21 @@ fn map_method_return<T: Serialize>(
 impl scenegraph::Scenegraph for Scenegraph {
 	fn send_signal(
 		&self,
-		node: u64,
+		node_id: u64,
+		aspect_id: u64,
 		method: u64,
 		data: &[u8],
 		fds: Vec<OwnedFd>,
 	) -> Result<(), ScenegraphError> {
 		let Some(client) = self.get_client() else {
-			return Err(ScenegraphError::SignalNotFound);
+			return Err(ScenegraphError::NodeNotFound);
 		};
-		debug_span!("Handle signal", node, method).in_scope(|| {
-			self.get_node(node)
+		debug_span!("Handle signal", aspect_id, node_id, method).in_scope(|| {
+			self.get_node(node_id)
 				.ok_or(ScenegraphError::NodeNotFound)?
 				.send_local_signal(
 					client,
+					aspect_id,
 					method,
 					Message {
 						data: data.to_vec(),
@@ -109,23 +110,25 @@ impl scenegraph::Scenegraph for Scenegraph {
 	}
 	fn execute_method(
 		&self,
-		node: u64,
+		node_id: u64,
+		aspect_id: u64,
 		method: u64,
 		data: &[u8],
 		fds: Vec<OwnedFd>,
 		response: oneshot::Sender<Result<(Vec<u8>, Vec<OwnedFd>), ScenegraphError>>,
 	) {
 		let Some(client) = self.get_client() else {
-			let _ = response.send(Err(ScenegraphError::MethodNotFound));
+			let _ = response.send(Err(ScenegraphError::NodeNotFound));
 			return;
 		};
-		debug!(node, method, "Handle method");
-		let Some(node) = self.get_node(node) else {
+		debug!(aspect_id, node_id, method, "Handle method");
+		let Some(node) = self.get_node(node_id) else {
 			let _ = response.send(Err(ScenegraphError::NodeNotFound));
 			return;
 		};
 		node.execute_local_method(
 			client,
+			aspect_id,
 			method,
 			Message {
 				data: data.to_vec(),

src/main.rs

@@ -1,40 +1,83 @@
 #![allow(clippy::empty_docs)]
+#![allow(clippy::too_many_arguments)]
+#![allow(clippy::type_complexity)]
 mod core;
 mod nodes;
 mod objects;
 mod session;
+pub mod tracking_offset;
 #[cfg(feature = "wayland")]
 mod wayland;
 
-use crate::core::destroy_queue;
-use crate::nodes::items::camera;
-use crate::nodes::{audio, drawable, input};
+use crate::nodes::input;
 
+use bevy::MinimalPlugins;
+use bevy::a11y::AccessibilityPlugin;
+use bevy::app::{App, ScheduleRunnerPlugin, TerminalCtrlCHandlerPlugin};
+use bevy::asset::{AssetMetaCheck, UnapprovedPathMode};
+use bevy::audio::AudioPlugin;
+use bevy::core_pipeline::CorePipelinePlugin;
+use bevy::core_pipeline::oit::OrderIndependentTransparencySettings;
+use bevy::diagnostic::DiagnosticsPlugin;
+use bevy::ecs::schedule::{ExecutorKind, ScheduleLabel};
+use bevy::gizmos::GizmoPlugin;
+use bevy::gltf::GltfPlugin;
+use bevy::input::InputPlugin;
+use bevy::pbr::PbrPlugin;
+use bevy::render::settings::{Backends, RenderCreation, WgpuSettings};
+use bevy::render::{RenderDebugFlags, RenderPlugin};
+use bevy::scene::ScenePlugin;
+use bevy::window::{CompositeAlphaMode, PresentMode};
+use bevy::winit::{WakeUp, WinitPlugin};
+use bevy_dmabuf::import::DmabufImportPlugin;
+use bevy_mod_openxr::action_set_attaching::OxrActionAttachingPlugin;
+use bevy_mod_openxr::action_set_syncing::OxrActionSyncingPlugin;
+use bevy_mod_openxr::add_xr_plugins;
+use bevy_mod_openxr::exts::OxrExtensions;
+use bevy_mod_openxr::features::overlay::OxrOverlaySettings;
+use bevy_mod_openxr::graphics::{GraphicsBackend, OxrManualGraphicsConfig};
+use bevy_mod_openxr::init::{OxrInitPlugin, should_run_frame_loop};
+use bevy_mod_openxr::reference_space::OxrReferenceSpacePlugin;
+use bevy_mod_openxr::render::{OxrRenderPlugin, OxrWaitFrameSystem};
+use bevy_mod_openxr::resources::{OxrFrameState, OxrFrameWaiter, OxrSessionConfig};
+use bevy_mod_openxr::types::AppInfo;
+use bevy_mod_xr::camera::XrProjection;
+use bevy_mod_xr::session::{XrFirst, XrHandleEvents, XrSessionPlugin};
 use clap::Parser;
-use core::client::Client;
+use core::client::{Client, tick_internal_client};
+use core::entity_handle::EntityHandlePlugin;
 use core::task;
 use directories::ProjectDirs;
-use objects::ServerObjects;
-use once_cell::sync::OnceCell;
+use nodes::audio::AudioNodePlugin;
+use nodes::drawable::lines::LinesNodePlugin;
+use nodes::drawable::model::ModelNodePlugin;
+use nodes::drawable::text::TextNodePlugin;
+use nodes::fields::FieldDebugGizmoPlugin;
+use nodes::spatial::SpatialNodePlugin;
+use objects::hmd::HmdPlugin;
+use objects::input::mouse_pointer::FlatscreenInputPlugin;
+use objects::input::oxr_controller::ControllerPlugin;
+use objects::input::oxr_hand::HandPlugin;
+use objects::play_space::PlaySpacePlugin;
+use openxr::{EnvironmentBlendMode, ReferenceSpaceType};
 use session::{launch_start, save_session};
+use stardust_xr::schemas::dbus::object_registry::ObjectRegistry;
 use stardust_xr::server;
+use std::ops::DerefMut as _;
 use std::path::PathBuf;
-use std::sync::Arc;
-use std::time::Duration;
-use stereokit_rust::material::Material;
-use stereokit_rust::shader::Shader;
-use stereokit_rust::sk::{sk_quit, AppMode, DepthMode, OriginMode, QuitReason, SkSettings};
-use stereokit_rust::system::{LogLevel, Renderer};
-use stereokit_rust::tex::{SHCubemap, Tex, TexFormat, TexType};
-use stereokit_rust::ui::Ui;
-use stereokit_rust::util::{Color128, Time};
+use std::sync::{Arc, OnceLock};
 use tokio::net::UnixListener;
 use tokio::sync::Notify;
+use tokio::task::JoinError;
 use tracing::metadata::LevelFilter;
-use tracing::{debug_span, error, info};
-use tracing_subscriber::{fmt, prelude::*, EnvFilter};
-use zbus::fdo::ObjectManager;
+use tracing::{error, info};
+use tracing_subscriber::{EnvFilter, fmt, prelude::*};
+use tracking_offset::TrackingOffsetPlugin;
+use wayland::{Wayland, WaylandPlugin};
 use zbus::Connection;
+use zbus::fdo::ObjectManager;
+
+use bevy::prelude::*;
 
 #[derive(Debug, Clone, Parser)]
 #[clap(author, version, about, long_about = None)]
@@ -43,6 +86,10 @@ struct CliArgs {
 	#[clap(short, long, action)]
 	flatscreen: bool,
 
+	/// Creates a transparent window fot the flatscreen mode
+	#[clap(short, long, action)]
+	transparent_flatscreen: bool,
+
 	/// If monado insists on emulating them, set this flag...we want the raw input
 	#[clap(long)]
 	disable_controllers: bool,
@@ -67,11 +114,13 @@ struct CliArgs {
 	restore: Option<String>,
 }
 
-static STARDUST_INSTANCE: OnceCell<String> = OnceCell::new();
+pub type BevyMaterial = StandardMaterial;
 
-// #[tokio::main]
-#[tokio::main(flavor = "current_thread")]
-async fn main() {
+static STARDUST_INSTANCE: OnceLock<String> = OnceLock::new();
+
+// #[tokio::main(flavor = "current_thread")]
+#[tokio::main]
+async fn main() -> Result<AppExit, JoinError> {
 	color_eyre::install().unwrap();
 
 	let registry = tracing_subscriber::registry();
@@ -91,7 +140,11 @@ async fn main() {
 		.with_thread_names(true)
 		.with_ansi(true)
 		.with_line_number(true)
-		.with_filter(EnvFilter::from_default_env());
+		.with_filter(
+			EnvFilter::builder()
+				.with_default_directive(LevelFilter::WARN.into())
+				.from_env_lossy(),
+		);
 	registry.with(log_layer).init();
 
 	let cli_args = CliArgs::parse();
@@ -120,16 +173,22 @@ async fn main() {
 
 	let project_dirs = ProjectDirs::from("", "", "stardust");
 	if project_dirs.is_none() {
-		error!("Unable to get Stardust project directories, default skybox and startup script will not work.");
+		error!(
+			"Unable to get Stardust project directories, default skybox and startup script will not work."
+		);
 	}
 
 	let dbus_connection = Connection::session()
 		.await
 		.expect("Could not open dbus session");
+	// why is this requested here? should there be a specific server bus name that we check
+	// instead?
 	dbus_connection
 		.request_name("org.stardustxr.HMD")
 		.await
-		.expect("Another instance of the server is running. This is not supported currently (but is planned).");
+		.expect(
+			"Another instance of the server is running. This is not supported currently (but is planned).",
+		);
 
 	dbus_connection
 		.object_server()
@@ -137,24 +196,34 @@ async fn main() {
 		.await
 		.expect("Couldn't add the object manager");
 
-	let sk_ready_notifier = Arc::new(Notify::new());
-	let stereokit_loop = tokio::task::spawn_blocking({
-		let sk_ready_notifier = sk_ready_notifier.clone();
+	let object_registry = ObjectRegistry::new(&dbus_connection).await.expect(
+		"Couldn't make the object registry to find all objects with given interfaces in d-bus",
+	);
+
+	let _wayland = Wayland::new().expect("Couldn't create Wayland instance");
+
+	let ready_notifier = Arc::new(Notify::new());
+	let io_loop = tokio::task::spawn_blocking({
+		let ready_notifier = ready_notifier.clone();
 		let project_dirs = project_dirs.clone();
 		let cli_args = cli_args.clone();
 		let dbus_connection = dbus_connection.clone();
-		move || stereokit_loop(sk_ready_notifier, project_dirs, cli_args, dbus_connection)
+		move || {
+			bevy_loop(
+				ready_notifier,
+				project_dirs,
+				cli_args,
+				dbus_connection,
+				object_registry,
+			)
+		}
 	});
-	sk_ready_notifier.notified().await;
+	ready_notifier.notified().await;
 	let mut startup_children = project_dirs
 		.as_ref()
 		.map(|project_dirs| launch_start(&cli_args, project_dirs))
 		.unwrap_or_default();
-
-	tokio::select! {
-		_ = stereokit_loop => (),
-		_ = tokio::signal::ctrl_c() => unsafe {sk_quit(QuitReason::SystemClose)},
-	}
+	let return_value = io_loop.await;
 	info!("Stopping...");
 	if let Some(project_dirs) = project_dirs {
 		save_session(&project_dirs).await;
@@ -164,126 +233,280 @@ async fn main() {
 	}
 
 	info!("Cleanly shut down Stardust");
+	return_value
 }
 
-fn stereokit_loop(
-	sk_ready_notifier: Arc<Notify>,
-	project_dirs: Option<ProjectDirs>,
+// static DEFAULT_SKYTEX: OnceLock<Tex> = OnceLock::new();
+// static DEFAULT_SKYLIGHT: OnceLock<SphericalHarmonics> = OnceLock::new();
+
+#[derive(ScheduleLabel, Hash, Debug, PartialEq, Eq, Clone, Copy)]
+pub struct PreFrameWait;
+#[derive(Resource, Deref)]
+pub struct ObjectRegistryRes(ObjectRegistry);
+#[derive(Resource, Deref)]
+pub struct DbusConnection(Connection);
+
+fn bevy_loop(
+	ready_notifier: Arc<Notify>,
+	_project_dirs: Option<ProjectDirs>,
 	args: CliArgs,
 	dbus_connection: Connection,
-) {
-	let sk = SkSettings::default()
-		.app_name("Stardust XR")
-		.mode(if args.flatscreen {
-			AppMode::Simulator
-		} else {
-			AppMode::XR
+	object_registry: ObjectRegistry,
+) -> AppExit {
+	let mut app = App::new();
+	app.insert_resource(DbusConnection(dbus_connection));
+	app.insert_resource(OxrManualGraphicsConfig {
+		fallback_backend: GraphicsBackend::Vulkan(()),
+		vk_instance_exts: Vec::new(),
+		vk_device_exts: bevy_dmabuf::required_device_extensions(),
+	});
+	app.add_plugins(AssetPlugin {
+		meta_check: AssetMetaCheck::Never,
+		unapproved_path_mode: UnapprovedPathMode::Allow,
+		..default()
+	});
+	let mut plugins = MinimalPlugins
+		.build()
+		.add(DiagnosticsPlugin)
+		.add(TransformPlugin)
+		.add(InputPlugin)
+		.add(AccessibilityPlugin);
+	plugins = plugins
+		.add(TerminalCtrlCHandlerPlugin)
+		// bevy_mod_openxr will replace this, TODO: figure out how to mix this with
+		// bevy-dmabuf
+		.add(RenderPlugin {
+			render_creation: RenderCreation::Automatic(WgpuSettings {
+				backends: Some(Backends::VULKAN),
+				..Default::default()
+			}),
+			..Default::default()
 		})
-		.depth_mode(DepthMode::D32)
-		.log_filter(match EnvFilter::from_default_env().max_level_hint() {
-			Some(LevelFilter::ERROR) => LogLevel::Error,
-			Some(LevelFilter::WARN) => LogLevel::Warning,
-			Some(LevelFilter::INFO) => LogLevel::Inform,
-			Some(LevelFilter::DEBUG) => LogLevel::Diagnostic,
-			Some(LevelFilter::TRACE) => LogLevel::Diagnostic,
-			Some(LevelFilter::OFF) => LogLevel::None,
-			None => LogLevel::Warning,
+		.add(ImagePlugin::default())
+		.add(CorePipelinePlugin)
+		// theoretically we shouldn't need this because of bevy_sk, but everything is tangled in
+		// there and idk what we actually need to run
+		.add(PbrPlugin {
+			// this seems to only apply to StandardMaterial, we don't use that
+			prepass_enabled: true,
+			add_default_deferred_lighting_plugin: false,
+			use_gpu_instance_buffer_builder: true,
+			debug_flags: RenderDebugFlags::default(),
 		})
-		.overlay_app(args.overlay_priority.is_some())
-		.overlay_priority(args.overlay_priority.unwrap_or(u32::MAX))
-		.disable_desktop_input_window(true)
-		.origin(OriginMode::Local)
-		.init()
-		.expect("StereoKit failed to initialize");
-	info!("Init StereoKit");
-
-	Renderer::multisample(0);
-	Material::default().shader(Shader::pbr_clip());
-	Ui::enable_far_interact(false);
-
-	// Skytex/light stuff
+		// required for gltf
+		.add(ScenePlugin)
+		.add(GltfPlugin::default())
+		// .add(AnimationPlugin)
+		.add(AudioPlugin::default())
+		.add(GizmoPlugin)
+		.add(WindowPlugin::default())
+		.add(DmabufImportPlugin);
+	let mut task_pool_plugin = TaskPoolPlugin::default();
+	// make tokio work
+	let handle = tokio::runtime::Handle::current();
+	let enter_runtime_context = Arc::new(move || {
+		// TODO: this might be a memory leak
+		std::mem::forget(handle.enter());
+	});
+	task_pool_plugin.task_pool_options.io.on_thread_spawn = Some(enter_runtime_context.clone());
+	task_pool_plugin.task_pool_options.compute.on_thread_spawn =
+		Some(enter_runtime_context.clone());
+	task_pool_plugin
+		.task_pool_options
+		.async_compute
+		.on_thread_spawn = Some(enter_runtime_context.clone());
+	plugins = plugins.set(task_pool_plugin);
+	if args.flatscreen
+		|| std::env::var_os("DISPLAY").is_some_and(|s| !s.is_empty())
+		|| std::env::var_os("WAYLAND_DISPLAY").is_some_and(|s| !s.is_empty())
 	{
-		if let Some(sky) = project_dirs
-			.as_ref()
-			.map(|dirs| dirs.config_dir().join("skytex.hdr"))
-			.filter(|f| f.exists())
-			.and_then(|p| SHCubemap::from_cubemap_equirectangular(p, true, 100).ok())
-		{
-			sky.render_as_sky();
+		let mut plugin = WinitPlugin::<WakeUp>::default();
+		plugin.run_on_any_thread = true;
+		plugins = plugins
+			.add(plugin)
+			.disable::<ScheduleRunnerPlugin>()
+			.add(FlatscreenInputPlugin);
+	}
+	app.add_plugins(
+		if !args.flatscreen {
+			add_xr_plugins(plugins)
+				.set(OxrInitPlugin {
+					app_info: AppInfo {
+						name: "Stardust XR".into(),
+						version: bevy_mod_openxr::types::Version(0, 44, 1),
+					},
+					exts: {
+						// all OpenXR extensions can be requested here
+						let mut exts = OxrExtensions::default();
+						exts.enable_hand_tracking();
+						if args.overlay_priority.is_some() {
+							exts.enable_extx_overlay();
+						}
+						exts.khr_convert_timespec_time = true;
+						exts
+					},
+					..default()
+				})
+				.set(OxrRenderPlugin {
+					default_wait_frame: false,
+					..default()
+				})
+				.set(OxrReferenceSpacePlugin {
+					default_primary_ref_space: ReferenceSpaceType::LOCAL,
+				})
+				// Disable a bunch of unneeded plugins
+				// we don't do any action stuff that needs to integrate with the ecosystem
+				.disable::<OxrActionAttachingPlugin>()
+				.disable::<OxrActionSyncingPlugin>()
 		} else {
-			Renderer::skytex(Tex::gen_color(
-				Color128::BLACK,
-				1,
-				1,
-				TexType::Cubemap,
-				TexFormat::RGBA32,
-			));
+			// enable a event
+			plugins = plugins.add(XrSessionPlugin { auto_handle: false });
+			bevy_dmabuf::wgpu_init::add_dmabuf_init_plugin(plugins)
 		}
-	}
-
-	#[cfg(feature = "wayland")]
-	let mut wayland = wayland::Wayland::new().expect("Could not initialize wayland");
-	#[cfg(feature = "wayland")]
-	wayland.make_context_current();
-	sk_ready_notifier.notify_waiters();
-	info!("Stardust ready!");
-
-	let mut objects = ServerObjects::new(
-		dbus_connection.clone(),
-		&sk,
-		args.disable_controllers,
-		args.disable_hands,
+		.set(WindowPlugin {
+			primary_window: Some(Window {
+				transparent: args.transparent_flatscreen,
+				present_mode: PresentMode::AutoNoVsync,
+				composite_alpha_mode: if args.transparent_flatscreen {
+					CompositeAlphaMode::PreMultiplied
+				} else {
+					CompositeAlphaMode::Inherit
+				},
+				title: "StardustXR server flatscreen mode".to_string(),
+				..default()
+			}),
+			..default()
+		}),
 	);
 
-	let mut last_frame_delta = Duration::ZERO;
-	let mut sleep_duration = Duration::ZERO;
-	while let Some(token) = sk.step() {
-		let _span = debug_span!("StereoKit step");
-		let _span = _span.enter();
-
-		camera::update(token);
-		#[cfg(feature = "wayland")]
-		wayland.frame_event();
-		destroy_queue::clear();
-
-		objects.update(&sk, token);
-		input::process_input();
-		nodes::root::Root::send_frame_events(Time::get_step_unscaled());
-		adaptive_sleep(
-			&mut last_frame_delta,
-			&mut sleep_duration,
-			Duration::from_micros(250),
-		);
-
-		#[cfg(feature = "wayland")]
-		wayland.update();
-		drawable::draw(token);
-		audio::update();
-	}
-
-	info!("Cleanly shut down StereoKit");
-	#[cfg(feature = "wayland")]
-	drop(wayland);
-}
-
-fn adaptive_sleep(
-	last_frame_delta: &mut Duration,
-	sleep_duration: &mut Duration,
-	sleep_duration_increase: Duration,
-) {
-	let frame_delta = Duration::from_secs_f64(Time::get_step_unscaled());
-	if *last_frame_delta < frame_delta {
-		if let Some(frame_delta_delta) = frame_delta.checked_sub(*last_frame_delta) {
-			if let Some(new_sleep_duration) = sleep_duration.checked_sub(frame_delta_delta) {
-				*sleep_duration = new_sleep_duration;
-			}
-		}
-	} else {
-		*sleep_duration += sleep_duration_increase;
-	}
-
-	debug_span!("Sleep", ?sleep_duration, ?frame_delta, ?last_frame_delta).in_scope(|| {
-		*last_frame_delta = frame_delta;
-		std::thread::sleep(*sleep_duration); // to give clients a chance to even update anything before drawing
+	app.add_plugins(bevy_sk::hand::HandPlugin);
+	// app.add_plugins(HandGizmosPlugin);
+	app.world_mut().resource_mut::<AmbientLight>().brightness = 1000.0;
+	if let Some(priority) = args.overlay_priority {
+		app.insert_resource(OxrOverlaySettings {
+			session_layer_placement: priority,
+			..default()
 		});
 	}
+	app.insert_resource(OxrSessionConfig {
+		blend_mode_preference: vec![
+			EnvironmentBlendMode::ALPHA_BLEND,
+			EnvironmentBlendMode::ADDITIVE,
+			EnvironmentBlendMode::OPAQUE,
+		],
+		..default()
+	});
+	let mut pre_frame_wait = Schedule::new(PreFrameWait);
+	pre_frame_wait.set_executor_kind(ExecutorKind::MultiThreaded);
+	app.add_schedule(pre_frame_wait);
+	app.insert_resource(ClearColor(Color::BLACK.with_alpha(0.0)));
+	app.insert_resource(ObjectRegistryRes(object_registry));
+	#[cfg(feature = "bevy_debugging")]
+	{
+		use bevy::remote::{RemotePlugin, http::RemoteHttpPlugin};
+		app.add_plugins((RemotePlugin::default(), RemoteHttpPlugin::default()));
+	}
+	// the Stardust server plugins
+	// infra plugins
+	app.add_plugins(EntityHandlePlugin);
+	// node plugins
+	app.add_plugins((
+		SpatialNodePlugin,
+		ModelNodePlugin,
+		TextNodePlugin,
+		LinesNodePlugin,
+		AudioNodePlugin,
+	));
+	// object plugins
+	app.add_plugins((PlaySpacePlugin, HandPlugin, ControllerPlugin, HmdPlugin));
+	// feature plugins
+	app.add_plugins((WaylandPlugin, TrackingOffsetPlugin, FieldDebugGizmoPlugin));
+	app.add_systems(PostStartup, move || {
+		ready_notifier.notify_waiters();
+	});
+	app.add_observer(cam_observer);
+	app.add_systems(
+		XrFirst,
+		xr_step
+			.in_set(OxrWaitFrameSystem)
+			.in_set(XrHandleEvents::FrameLoop),
+	);
+
+	app.run()
+}
+
+fn cam_observer(
+	trigger: Trigger<OnAdd, Camera3d>,
+	mut query: Query<(&mut Projection, &mut Msaa), With<Camera3d>>,
+) {
+	let Ok((mut projection, mut msaa)) = query.get_mut(trigger.target()) else {
+		return;
+	};
+	info!("modifying cam");
+	match projection.deref_mut() {
+		Projection::Perspective(perspective_projection) => perspective_projection.near = 0.003,
+		Projection::Orthographic(orthographic_projection) => orthographic_projection.near = 0.003,
+		Projection::Custom(custom_projection) => {
+			if let Some(xr) = custom_projection.get_mut::<XrProjection>() {
+				xr.near = 0.003
+			} else {
+				error_once!("unknown custom camera projection");
+			}
+		}
+	}
+	*msaa = Msaa::Off;
+}
+
+fn add_oit(
+	mut commands: Commands,
+	cameras: Query<
+		Entity,
+		(
+			With<Camera3d>,
+			Without<OrderIndependentTransparencySettings>,
+		),
+	>,
+) {
+	for entity in &cameras {
+		commands
+			.entity(entity)
+			.insert(OrderIndependentTransparencySettings {
+				layer_count: 4,
+				alpha_threshold: 0.00,
+			});
+	}
+}
+
+fn xr_step(world: &mut World) {
+	// update things like the Xr input methods
+	world.run_schedule(PreFrameWait);
+	input::process_input();
+	let time = world.resource::<bevy::prelude::Time>().delta_secs_f64();
+	nodes::root::Root::send_frame_events(time);
+
+	// we are targeting the frame after the wait
+	if let Some(mut state) = world.get_resource_mut::<OxrFrameState>() {
+		state.predicted_display_time = openxr::Time::from_nanos(
+			state.predicted_display_time.as_nanos() + state.predicted_display_period.as_nanos(),
+		);
+	}
+
+	let should_wait = world
+		.run_system_cached(should_run_frame_loop)
+		.unwrap_or(false);
+	// we might want to do an adaptive sleep when not OpenXR waiting
+	if should_wait {
+		world.resource_scope::<OxrFrameWaiter, _>(|world, mut waiter| {
+			let state = waiter
+				.wait()
+				.inspect_err(|err| error!("failed to wait OpenXR frame: {err}"))
+				.ok();
+
+			if let Some(state) = state {
+				world.insert_resource(OxrFrameState(state));
+			}
+		});
+	}
+
+	tick_internal_client();
+}

src/nodes/alias.rs

@@ -1,6 +1,5 @@
-use super::{Aspect, Node};
-use crate::core::{client::Client, registry::Registry};
-use color_eyre::eyre::Result;
+use super::{Aspect, AspectIdentifier, Node};
+use crate::core::{client::Client, error::Result, registry::Registry};
 use std::{
 	ops::Add,
 	sync::{Arc, Weak},
@@ -68,8 +67,31 @@ impl Alias {
 		Ok(())
 	}
 }
+impl AspectIdentifier for Alias {
+	const ID: u64 = 0;
+}
 impl Aspect for Alias {
-	const NAME: &'static str = "Alias";
+	fn as_any(self: Arc<Self>) -> Arc<dyn std::any::Any + Send + Sync + 'static> {
+		self
+	}
+	fn run_signal(
+		&self,
+		_calling_client: Arc<Client>,
+		_node: Arc<Node>,
+		_signal: u64,
+		_message: super::Message,
+	) -> Result<(), stardust_xr::scenegraph::ScenegraphError> {
+		Ok(())
+	}
+	fn run_method(
+		&self,
+		_calling_client: Arc<Client>,
+		_node: Arc<Node>,
+		_method: u64,
+		_message: super::Message,
+		_response: crate::core::scenegraph::MethodResponseSender,
+	) {
+	}
 }
 
 pub fn get_original(node: Arc<Node>, stop_on_disabled: bool) -> Option<Arc<Node>> {
@@ -106,7 +128,7 @@ impl AliasList {
 			.into_iter()
 			.find(move |node| links_to(node.clone(), original.clone()))
 	}
-	pub fn get_from_aspect<A: Aspect>(&self, aspect: &A) -> Option<Arc<Node>> {
+	pub fn get_from_aspect<A: AspectIdentifier>(&self, aspect: &A) -> Option<Arc<Node>> {
 		self.0.get_valid_contents().into_iter().find(|node| {
 			let Some(node) = get_original(node.clone(), false) else {
 				return false;
@@ -114,13 +136,13 @@ impl AliasList {
 			let Ok(aspect2) = node.get_aspect::<A>() else {
 				return false;
 			};
-			Arc::as_ptr(&aspect2) == (aspect as *const A)
+			std::ptr::eq(Arc::as_ptr(&aspect2), aspect)
 		})
 	}
 	pub fn get_aliases(&self) -> Vec<Arc<Node>> {
 		self.0.get_valid_contents()
 	}
-	pub fn remove_aspect<A: Aspect>(&self, aspect: &A) {
+	pub fn remove_aspect<A: AspectIdentifier>(&self, aspect: &A) {
 		self.0.retain(|node| {
 			let Some(original) = get_original(node.clone(), false) else {
 				return false;
@@ -128,7 +150,7 @@ impl AliasList {
 			let Ok(aspect2) = original.get_aspect::<A>() else {
 				return false;
 			};
-			Arc::as_ptr(&aspect2) != (aspect as *const A)
+			!std::ptr::eq(Arc::as_ptr(&aspect2), aspect)
 		})
 	}
 }

src/nodes/audio.rs

@@ -1,32 +1,104 @@
-use super::{Aspect, Node};
+use super::spatial::SpatialNode;
+use super::{Aspect, AspectIdentifier, Node};
 use crate::core::client::Client;
-use crate::core::destroy_queue;
+use crate::core::entity_handle::EntityHandle;
+use crate::core::error::Result;
 use crate::core::registry::Registry;
 use crate::core::resource::get_resource_file;
-use crate::create_interface;
-use crate::nodes::spatial::SPATIAL_ASPECT_ALIAS_INFO;
-use crate::nodes::spatial::{Spatial, Transform};
-use color_eyre::eyre::{eyre, Result};
-use glam::{vec3, Vec4Swizzles};
-use once_cell::sync::OnceCell;
+use crate::nodes::spatial::{SPATIAL_ASPECT_ALIAS_INFO, Spatial, Transform};
+use bevy::audio::{PlaybackMode, Volume};
+use bevy_mod_openxr::session::OxrSession;
+use bevy_mod_xr::session::{XrPreDestroySession, XrSessionCreated};
+use bevy_mod_xr::spaces::XrSpace;
+use color_eyre::eyre::eyre;
 use parking_lot::Mutex;
 use stardust_xr::values::ResourceID;
 
-use std::ops::DerefMut;
-use std::sync::Arc;
+use bevy::prelude::*;
+use bevy::transform::components::Transform as BevyTransform;
+use std::sync::{Arc, OnceLock};
 use std::{ffi::OsStr, path::PathBuf};
-use stereokit_rust::sound::{Sound as SkSound, SoundInst};
+
+pub struct AudioNodePlugin;
+impl Plugin for AudioNodePlugin {
+	fn build(&self, app: &mut App) {
+		app.add_systems(Update, update_sound_event);
+		app.add_systems(XrSessionCreated, spawn_hmd_audio_listener);
+		app.add_systems(XrPreDestroySession, despawn_hmd_audio_listener);
+	}
+}
+
+fn despawn_hmd_audio_listener(mut cmds: Commands, session: Res<OxrSession>, res: Res<HmdListener>) {
+	cmds.remove_resource::<HmdListener>();
+	cmds.entity(res.0).despawn();
+	_ = session.destroy_space(res.1);
+}
+
+fn spawn_hmd_audio_listener(mut cmds: Commands, session: Res<OxrSession>) {
+	let space = session
+		.create_reference_space(openxr::ReferenceSpaceType::VIEW, BevyTransform::IDENTITY)
+		.unwrap();
+	let listener = cmds
+		.spawn((
+			Name::new("HMD audio listener"),
+			space.0,
+			SpatialListener::new(0.2),
+		))
+		.id();
+	cmds.insert_resource(HmdListener(listener, space.0));
+}
+#[derive(Resource)]
+struct HmdListener(Entity, XrSpace);
+fn update_sound_event(
+	mut cmds: Commands,
+	sinks: Query<&SpatialAudioSink>,
+	asset_server: Res<AssetServer>,
+) {
+	for sound in SOUND_REGISTRY.get_valid_contents() {
+		if sound.entity.get().is_none() {
+			let handle = asset_server.load(sound.pending_audio_path.as_path());
+			sound
+				.entity
+				.set(
+					cmds.spawn((
+						Name::new("Audio Node"),
+						SpatialNode(Arc::downgrade(&sound.spatial)),
+						AudioPlayer::new(handle),
+						PlaybackSettings {
+							mode: PlaybackMode::Once,
+							volume: Volume::Linear(sound.volume),
+							speed: 1.0,
+							paused: true,
+							muted: false,
+							spatial: true,
+							spatial_scale: None,
+						},
+					))
+					.id()
+					.into(),
+				)
+				.unwrap();
+		}
+		if let Some(sink) = sound.entity.get().and_then(|e| sinks.get(e.0).ok()) {
+			if sound.play.lock().take().is_some() {
+				sink.play();
+			}
+			if sound.stop.lock().take().is_some() {
+				sink.stop();
+			}
+		}
+	}
+}
 
 static SOUND_REGISTRY: Registry<Sound> = Registry::new();
 
 stardust_xr_server_codegen::codegen_audio_protocol!();
 pub struct Sound {
-	space: Arc<Spatial>,
+	spatial: Arc<Spatial>,
 
 	volume: f32,
 	pending_audio_path: PathBuf,
-	sk_sound: OnceCell<SkSound>,
-	instance: Mutex<Option<SoundInst>>,
+	entity: OnceLock<EntityHandle>,
 	stop: Mutex<Option<()>>,
 	play: Mutex<Option<()>>,
 }
@@ -39,40 +111,23 @@ impl Sound {
 		)
 		.ok_or_else(|| eyre!("Resource not found"))?;
 		let sound = Sound {
-			space: node.get_aspect::<Spatial>().unwrap().clone(),
+			spatial: node.get_aspect::<Spatial>().unwrap().clone(),
 			volume: 1.0,
 			pending_audio_path,
-			sk_sound: OnceCell::new(),
-			instance: Mutex::new(None),
+			entity: OnceLock::new(),
 			stop: Mutex::new(None),
 			play: Mutex::new(None),
 		};
 		let sound_arc = SOUND_REGISTRY.add(sound);
 		node.add_aspect_raw(sound_arc.clone());
-		<Sound as SoundAspect>::add_node_members(node);
 		Ok(sound_arc)
 	}
-
-	fn update(&self) {
-		let sound = self
-			.sk_sound
-			.get_or_init(|| SkSound::from_file(self.pending_audio_path.clone()).unwrap());
-		if self.stop.lock().take().is_some() {
-			if let Some(instance) = self.instance.lock().take() {
-				instance.stop();
-			}
-		}
-		if self.instance.lock().is_none() && self.play.lock().take().is_some() {
-			let instance = sound.play(vec3(0.0, 0.0, 0.0), Some(self.volume));
-			self.instance.lock().replace(instance);
-		}
-		if let Some(instance) = self.instance.lock().deref_mut() {
-			instance.position(self.space.global_transform().w_axis.xyz());
-		}
 }
+impl AspectIdentifier for Sound {
+	impl_aspect_for_sound_aspect_id! {}
 }
 impl Aspect for Sound {
-	const NAME: &'static str = "Sound";
+	impl_aspect_for_sound_aspect! {}
 }
 impl SoundAspect for Sound {
 	fn play(node: Arc<Node>, _calling_client: Arc<Client>) -> Result<()> {
@@ -88,22 +143,11 @@ impl SoundAspect for Sound {
 }
 impl Drop for Sound {
 	fn drop(&mut self) {
-		if let Some(sk_sound) = self.sk_sound.take() {
-			destroy_queue::add(sk_sound);
-		}
 		SOUND_REGISTRY.remove(self);
 	}
 }
 
-pub fn update() {
-	for sound in SOUND_REGISTRY.get_valid_contents() {
-		sound.update()
-	}
-}
-
-create_interface!(AudioInterface);
-struct AudioInterface;
-impl InterfaceAspect for AudioInterface {
+impl InterfaceAspect for Interface {
 	#[doc = "Create a sound node. WAV and MP3 are supported."]
 	fn create_sound(
 		_node: Arc<Node>,

src/nodes/data.rs

@@ -1,276 +0,0 @@
-use super::alias::AliasList;
-use super::fields::Field;
-use super::spatial::{parse_transform, Spatial};
-use super::{Alias, Aspect, Node};
-use crate::core::client::Client;
-use crate::core::registry::Registry;
-use crate::create_interface;
-use crate::nodes::fields::FIELD_ALIAS_INFO;
-use crate::nodes::spatial::Transform;
-use crate::nodes::spatial::SPATIAL_ASPECT_ALIAS_INFO;
-use color_eyre::eyre::{bail, ensure, eyre, Result};
-use lazy_static::lazy_static;
-use parking_lot::Mutex;
-use slotmap::{DefaultKey, Key, KeyData, SlotMap};
-use stardust_xr::schemas::flex::flexbuffers;
-use stardust_xr::values::Datamap;
-use std::sync::{Arc, Weak};
-
-lazy_static! {
-	pub static ref KEYMAPS: Mutex<SlotMap<DefaultKey, String>> = Mutex::new(SlotMap::default());
-}
-
-// TODO: probably just use d-bus for this stuff (custom protocol for exporting spatials as refs) because the mask stuff is just too confusing
-
-static PULSE_SENDER_REGISTRY: Registry<PulseSender> = Registry::new();
-pub static PULSE_RECEIVER_REGISTRY: Registry<PulseReceiver> = Registry::new();
-
-pub fn get_mask(datamap: &Datamap) -> Result<flexbuffers::MapReader<&[u8]>> {
-	flexbuffers::Reader::get_root(datamap.raw().as_slice())
-		.map_err(|_| eyre!("Mask is not a valid flexbuffer"))?
-		.get_map()
-		.map_err(|_| eyre!("Mask is not a valid map"))
-}
-pub fn mask_matches(mask_map_lesser: &Datamap, mask_map_greater: &Datamap) -> bool {
-	(|| -> Result<_> {
-		for key in get_mask(mask_map_lesser)?.iter_keys() {
-			let lesser_key = get_mask(mask_map_lesser)?.index(key)?;
-			let greater_key = get_mask(mask_map_greater)?.index(key)?;
-			// otherwise zero-length vectors don't count the same as a single type vector
-			if lesser_key.flexbuffer_type().is_heterogenous_vector()
-				&& lesser_key.as_vector().is_empty()
-				&& greater_key.flexbuffer_type().is_vector()
-			{
-				continue;
-			}
-			if !lesser_key.flexbuffer_type().is_null()
-				&& lesser_key.flexbuffer_type() != greater_key.flexbuffer_type()
-			{
-				return Err(flexbuffers::ReaderError::InvalidPackedType {}.into());
-			}
-		}
-		Ok(())
-	})()
-	.is_ok()
-}
-
-stardust_xr_server_codegen::codegen_data_protocol!();
-
-pub struct PulseSender {
-	node: Weak<Node>,
-	pub mask: Datamap,
-	aliases: AliasList,
-	field_aliases: AliasList,
-}
-impl PulseSender {
-	pub fn add_to(node: &Arc<Node>, mask: Datamap) -> Result<Arc<PulseSender>> {
-		let sender = PulseSender {
-			node: Arc::downgrade(node),
-			mask,
-			aliases: AliasList::default(),
-			field_aliases: AliasList::default(),
-		};
-
-		// <PulseSender as PulseSenderAspect>::add_node_members(node);
-		let sender = PULSE_SENDER_REGISTRY.add(sender);
-		node.add_aspect_raw(sender.clone());
-		for receiver in PULSE_RECEIVER_REGISTRY.get_valid_contents() {
-			sender.handle_new_receiver(&receiver);
-		}
-		Ok(sender.clone())
-	}
-	fn handle_new_receiver(&self, receiver: &PulseReceiver) {
-		if !mask_matches(&self.mask, &receiver.mask) {
-			return;
-		}
-		let Some(tx_node) = self.node.upgrade() else {
-			return;
-		};
-		let Some(tx_client) = tx_node.get_client() else {
-			return;
-		};
-		let Some(rx_node) = receiver.node.upgrade() else {
-			return;
-		};
-		// Receiver itself
-		let Ok(rx_alias) = Alias::create(
-			&rx_node,
-			&tx_client,
-			PULSE_RECEIVER_ASPECT_ALIAS_INFO.clone(),
-			Some(&self.aliases),
-		) else {
-			return;
-		};
-
-		// Receiver's field
-		let Ok(rx_field_alias) = Alias::create(
-			&rx_node
-				.get_aspect::<PulseReceiver>()
-				.unwrap()
-				.field
-				.spatial
-				.node()
-				.unwrap(),
-			&tx_client,
-			FIELD_ALIAS_INFO.clone(),
-			Some(&self.aliases),
-		) else {
-			return;
-		};
-
-		let _ = pulse_sender_client::new_receiver(&tx_node, &rx_alias, &rx_field_alias);
-	}
-
-	fn handle_drop_receiver(&self, receiver: &PulseReceiver) {
-		let Some(node) = receiver.node.upgrade() else {
-			return;
-		};
-		self.aliases.remove_aspect(receiver);
-		self.field_aliases.remove_aspect(receiver.field.as_ref());
-		let Some(tx_node) = self.node.upgrade() else {
-			return;
-		};
-		let _ = pulse_sender_client::drop_receiver(&tx_node, node.get_id());
-	}
-}
-impl Aspect for PulseSender {
-	const NAME: &'static str = "PulseSender";
-}
-impl PulseSenderAspect for PulseSender {}
-impl Drop for PulseSender {
-	fn drop(&mut self) {
-		PULSE_SENDER_REGISTRY.remove(self);
-	}
-}
-
-pub struct PulseReceiver {
-	pub node: Weak<Node>,
-	pub field: Arc<Field>,
-	pub mask: Datamap,
-}
-impl PulseReceiver {
-	pub fn add_to(
-		node: &Arc<Node>,
-		field: Arc<Field>,
-		mask: Datamap,
-	) -> Result<Arc<PulseReceiver>> {
-		let receiver = PulseReceiver {
-			node: Arc::downgrade(node),
-			field,
-			mask,
-		};
-		let receiver = PULSE_RECEIVER_REGISTRY.add(receiver);
-
-		<PulseReceiver as PulseReceiverAspect>::add_node_members(node);
-		node.add_aspect_raw(receiver.clone());
-		for sender in PULSE_SENDER_REGISTRY.get_valid_contents() {
-			sender.handle_new_receiver(&receiver);
-		}
-		Ok(receiver)
-	}
-}
-impl Aspect for PulseReceiver {
-	const NAME: &'static str = "PulseReceiver";
-}
-impl PulseReceiverAspect for PulseReceiver {
-	fn send_data(
-		node: Arc<Node>,
-		_calling_client: Arc<Client>,
-		sender: Arc<Node>,
-		data: Datamap,
-	) -> Result<()> {
-		let this_receiver = node.get_aspect::<PulseReceiver>().unwrap();
-
-		ensure!(
-			mask_matches(&this_receiver.mask, &data),
-			"Message ({data:?}) does not contain the same keys as the receiver's mask ({:?})",
-			this_receiver.mask
-		);
-		pulse_receiver_client::data(&node, &sender, &data)?;
-		Ok(())
-	}
-}
-impl Drop for PulseReceiver {
-	fn drop(&mut self) {
-		PULSE_RECEIVER_REGISTRY.remove(self);
-		for sender in PULSE_SENDER_REGISTRY.get_valid_contents() {
-			sender.handle_drop_receiver(self);
-		}
-	}
-}
-
-create_interface!(DataInterface);
-struct DataInterface;
-impl InterfaceAspect for DataInterface {
-	fn create_pulse_sender(
-		_node: Arc<Node>,
-		calling_client: Arc<Client>,
-		id: u64,
-		parent: Arc<Node>,
-		transform: Transform,
-		mask: Datamap,
-	) -> Result<()> {
-		get_mask(&mask)?;
-		let node = Node::from_id(&calling_client, id, true);
-		let parent = parent.get_aspect::<Spatial>()?;
-		let transform = transform.to_mat4(true, true, false);
-
-		let node = node.add_to_scenegraph()?;
-		Spatial::add_to(&node, Some(parent.clone()), transform, false);
-		PulseSender::add_to(&node, mask)?;
-		Ok(())
-	}
-
-	fn create_pulse_receiver(
-		_node: Arc<Node>,
-		calling_client: Arc<Client>,
-		id: u64,
-		parent: Arc<Node>,
-		transform: Transform,
-		field: Arc<Node>,
-		mask: Datamap,
-	) -> Result<()> {
-		get_mask(&mask)?;
-		let node = Node::from_id(&calling_client, id, true);
-		let parent = parent.get_aspect::<Spatial>()?;
-		let transform = parse_transform(transform, true, true, false);
-		let field = field.get_aspect::<Field>()?;
-
-		let node = node.add_to_scenegraph()?;
-		Spatial::add_to(&node, Some(parent.clone()), transform, false);
-		PulseReceiver::add_to(&node, field, mask)?;
-		Ok(())
-	}
-
-	async fn register_keymap(
-		_node: Arc<Node>,
-		_calling_client: Arc<Client>,
-		keymap: String,
-	) -> Result<u64> {
-		let mut keymaps = KEYMAPS.lock();
-		if let Some(found_keymap_id) = keymaps
-			.iter()
-			.filter(|(_k, v)| *v == &keymap)
-			.map(|(k, _v)| k)
-			.last()
-		{
-			return Ok(found_keymap_id.data().as_ffi());
-		}
-
-		let key = keymaps.insert(keymap);
-		Ok(key.data().as_ffi())
-	}
-
-	async fn get_keymap(
-		_node: Arc<Node>,
-		_calling_client: Arc<Client>,
-		keymap_id: u64,
-	) -> Result<String> {
-		let keymaps = KEYMAPS.lock();
-		let Some(keymap) = keymaps.get(KeyData::from_ffi(keymap_id).into()) else {
-			bail!("Could not find keymap. Try registering it")
-		};
-
-		Ok(keymap.clone())
-	}
-}

src/nodes/drawable/holdout.wgsl

@@ -0,0 +1,5 @@
+fn fragment(
+    in: VertexOutput
+) -> FragmentOutput {
+    return vec4(0.0);
+}

src/nodes/drawable/lines.rs

@@ -1,22 +1,238 @@
 use super::{Line, LinesAspect};
 use crate::{
-	core::{client::Client, registry::Registry},
-	nodes::{spatial::Spatial, Aspect, Node},
+	BevyMaterial,
+	core::{
+		client::Client, color::ColorConvert, entity_handle::EntityHandle, error::Result,
+		registry::Registry,
+	},
+	nodes::{
+		Node,
+		spatial::{Spatial, SpatialNode},
+	},
+};
+use bevy::{
+	asset::RenderAssetUsages,
+	prelude::*,
+	render::{
+		mesh::{Indices, MeshAabb, PrimitiveTopology},
+		primitives::Aabb,
+	},
 };
-use color_eyre::eyre::Result;
 use glam::Vec3;
 use parking_lot::Mutex;
-use prisma::Lerp;
-use std::{collections::VecDeque, sync::Arc};
-use stereokit_rust::{
-	maths::Bounds, sk::MainThreadToken, system::LinePoint as SkLinePoint, util::Color128,
+use std::sync::{
+	Arc, OnceLock,
+	atomic::{AtomicBool, Ordering},
 };
 
+pub struct LinesNodePlugin;
+
+impl Plugin for LinesNodePlugin {
+	fn build(&self, app: &mut App) {
+		app.add_systems(Update, build_line_mesh);
+	}
+}
+
+fn build_line_mesh(
+	mut meshes: ResMut<Assets<Mesh>>,
+	mut cmds: Commands,
+	mut materials: ResMut<Assets<BevyMaterial>>,
+) {
+	for lines in LINES_REGISTRY
+		.get_valid_contents()
+		.into_iter()
+		.filter(|l| l.gen_mesh.load(Ordering::Relaxed))
+	{
+		lines.gen_mesh.store(false, Ordering::Relaxed);
+		let mut vertex_positions = Vec::<Vec3>::new();
+		let mut vertex_normals = Vec::<Vec3>::new();
+		let mut vertex_colors = Vec::<[f32; 4]>::new();
+		let mut vertex_indices = Vec::<u32>::new();
+		let lines_data = lines.data.lock();
+		if lines_data.is_empty() {
+			*lines.bounds.lock() = Aabb::default();
+			match lines.entity.get() {
+				Some(e) => cmds.entity(**e),
+				None => {
+					let e = cmds.spawn((
+						Name::new("LinesNode"),
+						SpatialNode(Arc::downgrade(&lines.spatial)),
+					));
+					_ = lines.entity.set(e.id().into());
+					e
+				}
+			}
+			.remove::<Mesh3d>();
+			continue;
+		}
+
+		let mut indices_set = 0;
+		for line in lines_data.iter() {
+			let start_set = indices_set;
+			// Create a sliding window of points to process each segment of the line
+			// For cyclic lines: wraps around by connecting last point back to first
+			// For non-cyclic lines: handles endpoints with None values
+			let point_windows = {
+				let mut out = Vec::new();
+				let mut last = line.cyclic.then(|| line.points.last()).flatten();
+				let mut peekable = line.points.iter().peekable();
+				while let Some(curr) = peekable.next() {
+					let mut end = false;
+					// Determine the next point - either the next in sequence or
+					// for cyclic lines, wrap back to first point at the end
+					let next = match peekable.peek() {
+						Some(v) => Some(*v),
+						None => {
+							end = true;
+							line.cyclic.then(|| line.points.first()).flatten()
+						}
+					};
+
+					out.push((last, curr, next, end));
+					last = Some(curr);
+				}
+				out
+			};
+			for (last, curr, next, last_point) in point_windows {
+				let last_quat = last.map(|v| {
+					Quat::from_rotation_arc(
+						Vec3::Y,
+						(Vec3::from(curr.point) - Vec3::from(v.point)).normalize(),
+					)
+				});
+				let next_quat = next.map(|v| {
+					Quat::from_rotation_arc(
+						Vec3::Y,
+						(Vec3::from(v.point) - Vec3::from(curr.point)).normalize(),
+					)
+				});
+				let quat = match (last_quat, next_quat) {
+					(None, None) => {
+						error!("no previous or next point in line");
+						break;
+					}
+					(None, Some(next)) => next,
+					(Some(last), None) => last,
+					(Some(last), Some(next)) => last.lerp(next, 0.5),
+				};
+				let normals = [
+					Vec3::X,
+					Vec3::new(1., 0., 1.).normalize(),
+					Vec3::Z,
+					Vec3::new(-1., 0., 1.).normalize(),
+					Vec3::NEG_X,
+					Vec3::new(-1., 0., -1.).normalize(),
+					Vec3::NEG_Z,
+					Vec3::new(1., 0., -1.).normalize(),
+				]
+				.map(Vec3::normalize)
+				.map(|v| (quat * v));
+				let points = normals.map(|v| (v * curr.thickness) + Vec3::from(curr.point));
+				vertex_normals.extend(normals);
+				vertex_positions.extend(points);
+				vertex_colors.extend([curr.color.to_bevy().to_srgba().to_f32_array(); 8]);
+				// Only connect vertices between segments if this isn't the end point
+				if !last_point {
+					vertex_indices.extend(indices(indices_set));
+				}
+				indices_set += 1;
+			}
+			// Handle the connection between start and end points:
+			// - For cyclic lines: connect last segment back to first
+			// - For non-cyclic lines: add caps at both ends
+			if line.cyclic {
+				vertex_indices.extend(cyclic_indices(start_set, indices_set - 1));
+			} else {
+				vertex_indices.extend(cap_indices(start_set, false));
+				vertex_indices.extend(cap_indices(indices_set - 1, true));
+			}
+		}
+		let mut mesh = Mesh::new(
+			PrimitiveTopology::TriangleList,
+			RenderAssetUsages::RENDER_WORLD,
+		);
+		if vertex_colors.iter().flatten().any(|v| !v.is_finite()) {
+			panic!("vertex colors contains non finite float: {vertex_colors:#?}",);
+		}
+		if vertex_normals.iter().any(|v| !v.is_finite()) {
+			panic!("normals contains non finite dir: {vertex_normals:#?}",);
+		}
+		if vertex_normals.iter().any(|v| !v.is_normalized()) {
+			panic!("normals contains non normalized dir: {vertex_normals:#?}",);
+		}
+		if vertex_positions.iter().any(|v| !v.is_finite()) {
+			panic!("vertex positions contains non finite pos: {vertex_positions:#?}",);
+		}
+		mesh.insert_indices(Indices::U32(vertex_indices));
+		mesh.insert_attribute(Mesh::ATTRIBUTE_COLOR, vertex_colors);
+		mesh.insert_attribute(Mesh::ATTRIBUTE_NORMAL, vertex_normals);
+		mesh.insert_attribute(Mesh::ATTRIBUTE_POSITION, vertex_positions.clone());
+		if let Some(aabb) = mesh.compute_aabb() {
+			info!(?aabb);
+			*lines.bounds.lock() = aabb;
+		}
+
+		match lines.entity.get() {
+			Some(e) => cmds.entity(**e),
+			None => {
+				let e = cmds.spawn((
+					Name::new("LinesNode"),
+					SpatialNode(Arc::downgrade(&lines.spatial)),
+					MeshMaterial3d(materials.add(BevyMaterial {
+						base_color: Color::WHITE,
+						perceptual_roughness: 1.0,
+						// TODO: this should be Blend
+						alpha_mode: AlphaMode::Opaque,
+						..default()
+					})),
+				));
+				_ = lines.entity.set(e.id().into());
+				e
+			}
+		}
+		.insert(Mesh3d(meshes.add(mesh)));
+	}
+}
+
+const END_CAP_INDICES: [u32; 18] = [0, 1, 7, 7, 1, 2, 7, 2, 6, 6, 2, 3, 6, 3, 5, 5, 3, 4];
+fn cap_indices(set: u32, flip: bool) -> [u32; END_CAP_INDICES.len()] {
+	let mut out = END_CAP_INDICES.map(|v| v + (set * 8));
+	if flip {
+		out.reverse();
+	}
+	out
+}
+
+// const BASE: [u16; 6] = [0, 8, 1, 8, 9, 1];
+
+// Defines how vertices are connected between consecutive cross-sections to form the tube
+const INDICES: [u32; 48] = [
+	0, 8, 1, 8, 9, 1, 1, 9, 2, 9, 10, 2, 2, 10, 3, 10, 11, 3, 3, 11, 4, 11, 12, 4, 4, 12, 5, 12,
+	13, 5, 5, 13, 6, 13, 14, 6, 6, 14, 7, 14, 15, 7, 7, 15, 0, 15, 8, 0,
+];
+fn indices(set: u32) -> [u32; INDICES.len()] {
+	INDICES.map(|v| v + (set * 8))
+}
+fn cyclic_indices(start_set: u32, end_set: u32) -> [u32; INDICES.len()] {
+	let mut out = INDICES.map(|v| {
+		if v < 8 {
+			v + ((start_set) * 8)
+		} else {
+			v + ((end_set - 1) * 8)
+		}
+	});
+	out.reverse();
+	out
+}
+
 static LINES_REGISTRY: Registry<Lines> = Registry::new();
 
 pub struct Lines {
-	space: Arc<Spatial>,
+	spatial: Arc<Spatial>,
 	data: Mutex<Vec<Line>>,
+	gen_mesh: AtomicBool,
+	entity: OnceLock<EntityHandle>,
+	bounds: Mutex<Aabb>,
 }
 impl Lines {
 	pub fn add_to(node: &Arc<Node>, lines: Vec<Line>) -> Result<Arc<Lines>> {
@@ -25,71 +241,29 @@ impl Lines {
 			.unwrap()
 			.bounding_box_calc
 			.set(|node| {
-				let mut bounds = Bounds::default();
-				if let Ok(lines) = node.get_aspect::<Lines>() {
-					for line in &*lines.data.lock() {
-						for point in &line.points {
-							bounds.grown_point(Vec3::from(point.point));
-						}
-					}
-				}
-				bounds
+				node.get_aspect::<Lines>()
+					.ok()
+					.map(|v| *v.bounds.lock())
+					.unwrap_or_default()
 			});
 
 		let lines = LINES_REGISTRY.add(Lines {
-			space: node.get_aspect::<Spatial>()?.clone(),
+			spatial: node.get_aspect::<Spatial>()?.clone(),
 			data: Mutex::new(lines),
+			gen_mesh: AtomicBool::new(true),
+			entity: OnceLock::new(),
+			bounds: Mutex::new(Aabb::default()),
 		});
-		<Lines as LinesAspect>::add_node_members(node);
 		node.add_aspect_raw(lines.clone());
 
 		Ok(lines)
 	}
-
-	fn draw(&self, token: &MainThreadToken) {
-		let transform_mat = self.space.global_transform();
-		let data = self.data.lock().clone();
-		for line in &data {
-			let mut points: VecDeque<SkLinePoint> = line
-				.points
-				.iter()
-				.map(|p| SkLinePoint {
-					pt: transform_mat.transform_point3(Vec3::from(p.point)).into(),
-					thickness: p.thickness,
-					color: Color128::new(p.color.c.r, p.color.c.g, p.color.c.b, p.color.a).into(),
-				})
-				.collect();
-			if line.cyclic && !points.is_empty() {
-				let first = line.points.first().unwrap();
-				let last = line.points.last().unwrap();
-
-				let color = Color128 {
-					r: first.color.c.r.lerp(&last.color.c.r, 0.5),
-					g: first.color.c.g.lerp(&last.color.c.g, 0.5),
-					b: first.color.c.b.lerp(&last.color.c.b, 0.5),
-					a: first.color.a.lerp(&last.color.a, 0.5),
-				};
-				let connect_point = SkLinePoint {
-					pt: transform_mat
-						.transform_point3(Vec3::from(first.point).lerp(Vec3::from(last.point), 0.5))
-						.into(),
-					thickness: (first.thickness + last.thickness) * 0.5,
-					color: color.into(),
-				};
-				points.push_front(connect_point);
-				points.push_back(connect_point);
-			}
-			stereokit_rust::system::Lines::add_list(token, points.make_contiguous());
-		}
-	}
-}
-impl Aspect for Lines {
-	const NAME: &'static str = "Lines";
 }
 impl LinesAspect for Lines {
 	fn set_lines(node: Arc<Node>, _calling_client: Arc<Client>, lines: Vec<Line>) -> Result<()> {
 		let lines_aspect = node.get_aspect::<Lines>()?;
 		*lines_aspect.data.lock() = lines;
+		lines_aspect.gen_mesh.store(true, Ordering::Relaxed);
 		Ok(())
 	}
 }
@@ -98,13 +272,3 @@ impl Drop for Lines {
 		LINES_REGISTRY.remove(self);
 	}
 }
-
-pub fn draw_all(token: &MainThreadToken) {
-	for lines in LINES_REGISTRY.get_valid_contents() {
-		if let Some(node) = lines.space.node() {
-			if node.enabled() {
-				lines.draw(token);
-			}
-		}
-	}
-}

src/nodes/drawable/mod.rs

@@ -1,55 +1,62 @@
 pub mod lines;
 pub mod model;
-#[cfg(feature = "wayland")]
-pub mod shader_manipulation;
-pub mod shaders;
 pub mod text;
 
 use self::{lines::Lines, model::Model, text::Text};
 use super::{
+	Aspect, AspectIdentifier, Node,
 	spatial::{Spatial, Transform},
-	Node,
 };
+use crate::core::{client::Client, error::Result, resource::get_resource_file};
 use crate::nodes::spatial::SPATIAL_ASPECT_ALIAS_INFO;
-use crate::{
-	core::{client::Client, resource::get_resource_file},
-	create_interface,
-};
-use color_eyre::eyre::{self, Result};
+use color_eyre::eyre::eyre;
+use model::ModelPart;
 use parking_lot::Mutex;
 use stardust_xr::values::ResourceID;
 use std::{ffi::OsStr, path::PathBuf, sync::Arc};
-use stereokit_rust::{sk::MainThreadToken, system::Renderer, tex::SHCubemap};
 
-// #[instrument(level = "debug", skip(sk))]
-pub fn draw(token: &MainThreadToken) {
-	lines::draw_all(token);
-	model::draw_all(token);
-	text::draw_all(token);
-
-	if let Some(skytex) = QUEUED_SKYTEX.lock().take() {
-		if let Ok(skytex) = SHCubemap::from_cubemap_equirectangular(skytex, true, 100) {
-			Renderer::skytex(skytex.tex);
-		}
-	}
-	if let Some(skylight) = QUEUED_SKYLIGHT.lock().take() {
-		if let Ok(skylight) = SHCubemap::from_cubemap_equirectangular(skylight, true, 100) {
-			Renderer::skylight(skylight.sh);
-		}
-	}
-}
-
-static QUEUED_SKYLIGHT: Mutex<Option<PathBuf>> = Mutex::new(None);
-static QUEUED_SKYTEX: Mutex<Option<PathBuf>> = Mutex::new(None);
+static QUEUED_SKYLIGHT: Mutex<Option<Option<PathBuf>>> = Mutex::new(None);
+static QUEUED_SKYTEX: Mutex<Option<Option<PathBuf>>> = Mutex::new(None);
 
 stardust_xr_server_codegen::codegen_drawable_protocol!();
-create_interface!(DrawableInterface);
 
-pub struct DrawableInterface;
-impl InterfaceAspect for DrawableInterface {
-	fn set_sky_tex(_node: Arc<Node>, calling_client: Arc<Client>, tex: ResourceID) -> Result<()> {
-		let resource_path = get_resource_file(&tex, &calling_client, &[OsStr::new("hdr")])
-			.ok_or(eyre::eyre!("Could not find resource"))?;
+impl AspectIdentifier for Lines {
+	impl_aspect_for_lines_aspect_id! {}
+}
+impl Aspect for Lines {
+	impl_aspect_for_lines_aspect! {}
+}
+impl AspectIdentifier for Model {
+	impl_aspect_for_model_aspect_id! {}
+}
+impl Aspect for Model {
+	impl_aspect_for_model_aspect! {}
+}
+impl AspectIdentifier for ModelPart {
+	impl_aspect_for_model_part_aspect_id! {}
+}
+impl Aspect for ModelPart {
+	impl_aspect_for_model_part_aspect! {}
+}
+impl AspectIdentifier for Text {
+	impl_aspect_for_text_aspect_id! {}
+}
+impl Aspect for Text {
+	impl_aspect_for_text_aspect! {}
+}
+
+impl InterfaceAspect for Interface {
+	fn set_sky_tex(
+		_node: Arc<Node>,
+		calling_client: Arc<Client>,
+		tex: Option<ResourceID>,
+	) -> Result<()> {
+		let resource_path = tex
+			.map(|tex| {
+				get_resource_file(&tex, &calling_client, &[OsStr::new("hdr")])
+					.ok_or(eyre!("Could not find resource"))
+			})
+			.transpose()?;
 		QUEUED_SKYTEX.lock().replace(resource_path);
 		Ok(())
 	}
@@ -57,10 +64,14 @@ impl InterfaceAspect for DrawableInterface {
 	fn set_sky_light(
 		_node: Arc<Node>,
 		calling_client: Arc<Client>,
-		light: ResourceID,
+		light: Option<ResourceID>,
 	) -> Result<()> {
-		let resource_path = get_resource_file(&light, &calling_client, &[OsStr::new("hdr")])
-			.ok_or(eyre::eyre!("Could not find resource"))?;
+		let resource_path = light
+			.map(|light| {
+				get_resource_file(&light, &calling_client, &[OsStr::new("hdr")])
+					.ok_or(eyre!("Could not find resource"))
+			})
+			.transpose()?;
 		QUEUED_SKYLIGHT.lock().replace(resource_path);
 		Ok(())
 	}

src/nodes/drawable/model.rs

@@ -1,291 +1,484 @@
-use super::{MaterialParameter, ModelAspect, ModelPartAspect, MODEL_PART_ASPECT_ALIAS_INFO};
+use super::{MODEL_PART_ASPECT_ALIAS_INFO, MaterialParameter, ModelAspect, ModelPartAspect};
+use crate::core::bevy_channel::{BevyChannel, BevyChannelReader};
 use crate::core::client::Client;
+use crate::core::color::ColorConvert as _;
+use crate::core::entity_handle::EntityHandle;
+use crate::core::error::Result;
 use crate::core::registry::Registry;
 use crate::core::resource::get_resource_file;
+use crate::nodes::Node;
 use crate::nodes::alias::{Alias, AliasList};
-use crate::nodes::spatial::Spatial;
-use crate::nodes::{Aspect, Node};
-use color_eyre::eyre::{bail, eyre, Result};
-use glam::{Mat4, Vec2, Vec3};
-use once_cell::sync::{Lazy, OnceCell};
+use crate::nodes::spatial::{Spatial, SpatialNode};
+use crate::{BevyMaterial, bail};
+use bevy::asset::{load_internal_asset, weak_handle};
+use bevy::gltf::GltfLoaderSettings;
+use bevy::pbr::{ExtendedMaterial, MaterialExtension};
+use bevy::prelude::*;
+use bevy::render::primitives::Aabb;
+use bevy::render::render_resource::{AsBindGroup, ShaderRef};
+use color_eyre::eyre::eyre;
 use parking_lot::Mutex;
-use rustc_hash::FxHashMap;
+use rustc_hash::{FxHashMap, FxHasher};
 use stardust_xr::values::ResourceID;
 use std::ffi::OsStr;
 use std::hash::{Hash, Hasher};
-use std::sync::{Arc, Weak};
-use stereokit_rust::material::Transparency;
-use stereokit_rust::maths::Bounds;
-use stereokit_rust::sk::MainThreadToken;
-use stereokit_rust::{material::Material, model::Model as SKModel, tex::Tex, util::Color128};
+use std::path::PathBuf;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::{Arc, OnceLock, Weak};
 
-pub struct MaterialWrapper(pub Material);
-impl Hash for MaterialWrapper {
-	fn hash<H: Hasher>(&self, state: &mut H) {
-		self.0.get_shader().0.as_ptr().hash(state);
-		for param in self.0.get_all_param_info() {
-			param.to_string().hash(state)
-		}
-		self.0.get_chain().map(MaterialWrapper).hash(state)
-	}
-}
-impl PartialEq for MaterialWrapper {
-	fn eq(&self, other: &Self) -> bool {
-		if self.0.get_shader().0.as_ptr() != other.0.get_shader().0.as_ptr() {
-			return false;
-		}
-		if self.0.get_all_param_info().count() != other.0.get_all_param_info().count() {
-			return false;
-		}
-		for self_param in self.0.get_all_param_info() {
-			let Some(other_param) = other
-				.0
-				.get_all_param_info()
-				.get_data(self_param.get_name(), self_param.get_type())
-			else {
-				return false;
-			};
-			if self_param.to_string() != other_param.to_string() {
-				return false;
-			}
-		}
-		self.0.get_chain().map(MaterialWrapper) == other.0.get_chain().map(MaterialWrapper)
-	}
-}
-impl Eq for MaterialWrapper {}
-unsafe impl Send for MaterialWrapper {}
-unsafe impl Sync for MaterialWrapper {}
+static LOAD_MODEL: BevyChannel<(Arc<Model>, PathBuf)> = BevyChannel::new();
 
-#[derive(Default)]
-struct MaterialRegistry(Mutex<FxHashMap<u64, String>>);
-impl MaterialRegistry {
-	fn add_or_get(&self, material: Arc<MaterialWrapper>) -> Arc<MaterialWrapper> {
-		let mut lock = self.0.lock();
-		let hash = {
-			use std::hash::{Hash, Hasher};
-			let mut hasher = std::collections::hash_map::DefaultHasher::new();
-			material.hash(&mut hasher);
-			hasher.finish()
-		};
+type HoldoutMaterial = ExtendedMaterial<BevyMaterial, HoldoutExtension>;
+const HOLDOUT_SHADER_HANDLE: Handle<Shader> = weak_handle!("92b481b7-d3da-4188-b252-2335ec814ee2");
+const HOLDOUT_MATERIAL_HANDLE: Handle<HoldoutMaterial> =
+	weak_handle!("d56f1d62-9121-434b-a34f-9f0bbd6b3390");
 
-		if let Some(id) = lock.get(&hash) {
-			if let Ok(existing) = Material::find(id) {
-				return Arc::new(MaterialWrapper(existing));
-			}
-		}
+pub struct ModelNodePlugin;
+#[derive(SystemSet, Hash, Debug, PartialEq, Eq, Clone, Copy)]
+pub struct ModelNodeSystemSet;
+impl Plugin for ModelNodePlugin {
+	fn build(&self, app: &mut App) {
+		LOAD_MODEL.init(app);
 
-		lock.insert(hash, material.0.get_id().to_string());
-		material
-	}
-}
+		load_internal_asset!(
+			app,
+			HOLDOUT_SHADER_HANDLE,
+			"holdout.wgsl",
+			Shader::from_wgsl
+		);
+		app.add_plugins(MaterialPlugin::<HoldoutMaterial>::default());
+		app.world_mut()
+			.resource_mut::<Assets<HoldoutMaterial>>()
+			.insert(&HOLDOUT_MATERIAL_HANDLE, HoldoutMaterial::default());
 
-static MATERIAL_REGISTRY: Lazy<MaterialRegistry> = Lazy::new(MaterialRegistry::default);
-static MODEL_REGISTRY: Registry<Model> = Registry::new();
-static HOLDOUT_MATERIAL: OnceCell<Arc<MaterialWrapper>> = OnceCell::new();
-
-impl MaterialParameter {
-	fn apply_to_material(&self, client: &Client, material: &Material, parameter_name: &str) {
-		let mut params = material.get_all_param_info();
-		match self {
-			MaterialParameter::Bool(val) => {
-				params.set_bool(parameter_name, *val);
-			}
-			MaterialParameter::Int(val) => {
-				params.set_int(parameter_name, &[*val]);
-			}
-			MaterialParameter::UInt(val) => {
-				params.set_uint(parameter_name, &[*val]);
-			}
-			MaterialParameter::Float(val) => {
-				params.set_float(parameter_name, *val);
-			}
-			MaterialParameter::Vec2(val) => {
-				params.set_vec2(parameter_name, Vec2::from(*val));
-			}
-			MaterialParameter::Vec3(val) => {
-				params.set_vec3(parameter_name, Vec3::from(*val));
-			}
-			MaterialParameter::Color(val) => {
-				params.set_color(
-					parameter_name,
-					Color128::new(val.c.r, val.c.g, val.c.b, val.a),
+		app.init_resource::<MaterialRegistry>();
+		app.add_systems(
+			Update,
+			(
+				load_models,
+				gen_model_parts.after(TransformSystem::TransformPropagate),
+				apply_materials,
+			)
+				.chain()
+				.in_set(ModelNodeSystemSet),
 		);
 	}
+}
+
+// No extra data needed for a simple holdout
+#[derive(Default, Asset, AsBindGroup, TypePath, Debug, Clone)]
+pub struct HoldoutExtension {}
+impl MaterialExtension for HoldoutExtension {
+	fn fragment_shader() -> ShaderRef {
+		HOLDOUT_SHADER_HANDLE.into()
+	}
+
+	fn alpha_mode() -> Option<AlphaMode> {
+		Some(AlphaMode::Opaque)
+	}
+}
+
+#[derive(Component)]
+struct ModelNode(Weak<Model>);
+
+fn load_models(
+	asset_server: Res<AssetServer>,
+	mut cmds: Commands,
+	mut mpsc_receiver: ResMut<BevyChannelReader<(Arc<Model>, PathBuf)>>,
+) {
+	while let Some((model, path)) = mpsc_receiver.read() {
+		// idk of the asset label is the correct approach here
+		let handle = asset_server.load_with_settings(
+			GltfAssetLabel::Scene(0).from_asset(path),
+			|settings: &mut GltfLoaderSettings| {
+				settings.load_cameras = false;
+				settings.load_lights = false;
+			},
+		);
+		let entity = cmds
+			.spawn((
+				Name::new("ModelNode"),
+				SceneRoot(handle),
+				ModelNode(Arc::downgrade(&model)),
+				SpatialNode(Arc::downgrade(&model.spatial)),
+			))
+			.id();
+		model.bevy_scene_entity.set(entity.into()).unwrap();
+	}
+}
+
+fn apply_materials(
+	mut commands: Commands,
+	mut query: Query<&mut MeshMaterial3d<BevyMaterial>>,
+	mut material_registry: ResMut<MaterialRegistry>,
+	asset_server: Res<AssetServer>,
+	mut materials: ResMut<Assets<BevyMaterial>>,
+) -> bevy::prelude::Result {
+	for model_part in MODEL_REGISTRY
+		.get_valid_contents()
+		.iter()
+		.filter_map(|p| p.parts.get())
+		.flatten()
+	{
+		let entity = **model_part.mesh_entity.get().unwrap();
+		let Ok(mut mesh_mat) = query.get_mut(entity) else {
+			continue;
+		};
+		if model_part.holdout.load(Ordering::Relaxed) {
+			commands
+				.entity(entity)
+				.remove::<MeshMaterial3d<BevyMaterial>>()
+				.insert(MeshMaterial3d(HOLDOUT_MATERIAL_HANDLE));
+			continue;
+		}
+		if let Some(material) = model_part.pending_material_replacement.lock().take()
+			&& let Some(material) = materials.get(&material)
+		{
+			let handle = material_registry.get_handle(material.clone(), &mut materials);
+			mesh_mat.0 = handle;
+		}
+		for (param_name, param) in model_part.pending_material_parameters.lock().drain() {
+			let mut new_mat = materials.get(&mesh_mat.0).unwrap().clone();
+			param.apply_to_material(
+				&model_part.space.node().unwrap().get_client().unwrap(),
+				&mut new_mat,
+				&param_name,
+				&asset_server,
+			);
+			let handle = material_registry.get_handle(new_mat, &mut materials);
+			mesh_mat.0 = handle;
+		}
+	}
+
+	Ok(())
+}
+
+fn gen_model_parts(
+	scenes: Res<Assets<Scene>>,
+	query: Query<(&SceneRoot, &ModelNode, &Children)>,
+	children_query: Query<&Children>,
+	part_query: Query<(&Name, Option<&Children>, &Transform), Without<Mesh3d>>,
+	part_mesh_query: Query<(&Transform, &Aabb), With<Mesh3d>>,
+	has_mesh: Query<Has<Mesh3d>>,
+	mut cmds: Commands,
+) {
+	for (scene_root, model_node, model_children) in query.iter() {
+		let Some(model) = model_node.0.upgrade() else {
+			continue;
+		};
+		if model.parts.get().is_some() {
+			continue;
+		}
+		if scenes.get(scene_root.0.id()).is_none() {
+			continue;
+		}
+		let mut parts = Vec::new();
+		for entity in model_children
+			.iter()
+			.filter_map(|e| children_query.get(e).ok())
+			.flat_map(|c| c.iter())
+		{
+			gen_path(
+				entity,
+				&part_query,
+				None,
+				&mut |entity, name, transform, parent, children| {
+					let path = parent
+						.as_ref()
+						.map(|p| format!("{}/{}", &p.path, name.as_str()))
+						.unwrap_or_else(|| name.to_string());
+					let parent_spatial = parent
+						.as_ref()
+						.map(|p| p.space.clone())
+						.unwrap_or_else(|| model.spatial.clone());
+					let client = model.spatial.node()?.get_client()?;
+					let (spatial, model_part) =
+						match model.pre_bound_parts.lock().iter().find(|v| v.path == path) {
+							None => {
+								let node =
+									client.scenegraph.add_node(Node::generate(&client, false));
+								let spatial = Spatial::add_to(
+									&node,
+									Some(parent_spatial),
+									transform.compute_matrix(),
+									false,
+								);
+								let model_part = node.add_aspect(ModelPart {
+									entity: OnceLock::new(),
+									mesh_entity: OnceLock::new(),
+									path,
+									space: spatial.clone(),
+									_model: Arc::downgrade(&model),
+									pending_material_parameters: Mutex::default(),
+									pending_material_replacement: Mutex::default(),
+									holdout: AtomicBool::new(false),
+									aliases: AliasList::default(),
+									bounds: OnceLock::new(),
+								});
+								(spatial, model_part)
+							}
+							Some(part) => {
+								part.space.set_spatial_parent(&parent_spatial).unwrap();
+								(part.space.clone(), part.clone())
+							}
+						};
+					let aabb = Aabb::enclosing(
+						children
+							.iter()
+							.flat_map(|v| v.iter())
+							.filter_map(|e| part_mesh_query.get(e).ok())
+							.flat_map(|(transform, aabb)| {
+								[
+									transform.transform_point(aabb.min().into()),
+									transform.transform_point(aabb.max().into()),
+								]
+							}),
+					)
+					.unwrap_or_default();
+					_ = spatial.bounding_box_calc.set(move |n| {
+						n.get_aspect::<ModelPart>()
+							.ok()
+							.and_then(|v| v.bounds.get().copied())
+							.unwrap_or_default()
+					});
+					spatial.set_local_transform(transform.compute_matrix());
+
+					cmds.entity(entity)
+						.insert(SpatialNode(Arc::downgrade(&spatial)));
+					let mesh_entity = children_query
+						.get(entity)
+						.iter()
+						.flat_map(|v| v.iter())
+						.find(|e| has_mesh.get(*e).unwrap_or(false))?;
+					_ = model_part.bounds.set(aabb);
+					_ = model_part.entity.set(entity.into());
+					_ = model_part.mesh_entity.set(mesh_entity.into());
+					parts.push(model_part.clone());
+					Some(model_part)
+				},
+			);
+		}
+		_ = model.parts.set(parts);
+	}
+}
+
+fn gen_path(
+	current_entity: Entity,
+	part_query: &Query<(&Name, Option<&Children>, &Transform), Without<Mesh3d>>,
+	parent: Option<Arc<ModelPart>>,
+	func: &mut dyn FnMut(
+		Entity,
+		&Name,
+		&Transform,
+		Option<Arc<ModelPart>>,
+		Option<&Children>,
+	) -> Option<Arc<ModelPart>>,
+) {
+	let Ok((name, children, transform)) = part_query.get(current_entity) else {
+		return;
+	};
+	let Some(parent) = func(current_entity, name, transform, parent, children) else {
+		return;
+	};
+	for e in children.iter().flat_map(|c| c.iter()) {
+		gen_path(e, part_query, Some(parent.clone()), func);
+	}
+}
+
+#[derive(PartialEq, Deref, DerefMut, Clone, Copy, Eq, PartialOrd, Ord, Hash)]
+struct HashedPbrMaterial(u64);
+impl HashedPbrMaterial {
+	fn new(material: &BevyMaterial) -> Self {
+		let mut hasher = FxHasher::default();
+		Self::hash_pbr_mat(material, &mut hasher);
+		Self(hasher.finish())
+	}
+	fn hash_pbr_mat<H: Hasher>(mat: &BevyMaterial, state: &mut H) {
+		hash_color(mat.base_color, state);
+		hash_color(mat.emissive.into(), state);
+		state.write_u32(mat.metallic.to_bits());
+		state.write_u32(mat.perceptual_roughness.to_bits());
+		match mat.alpha_mode {
+			AlphaMode::Opaque => state.write_u8(0),
+			AlphaMode::Mask(v) => {
+				state.write_u8(1);
+				state.write_u32(v.to_bits());
+			}
+			AlphaMode::Blend => state.write_u8(2),
+			AlphaMode::Premultiplied => state.write_u8(3),
+			AlphaMode::AlphaToCoverage => state.write_u8(4),
+			AlphaMode::Add => state.write_u8(5),
+			AlphaMode::Multiply => state.write_u8(6),
+		}
+		state.write_u8(mat.double_sided as u8);
+		mat.base_color_texture.hash(state);
+		mat.emissive_texture.hash(state);
+		mat.metallic_roughness_texture.hash(state);
+		mat.occlusion_texture.hash(state);
+		// should always be the same, TODO: make the spherical harmonics buffer a per mesh instance thing
+		// mat.spherical_harmonics.hash(state);
+	}
+}
+fn hash_color<H: Hasher>(color: Color, state: &mut H) {
+	match color {
+		Color::Srgba(srgba) => {
+			state.write_u8(0);
+			state.write(&srgba.to_u8_array());
+		}
+		Color::LinearRgba(linear_rgba) => {
+			state.write_u8(1);
+			state.write(&linear_rgba.to_u8_array());
+		}
+		Color::Hsla(hsla) => {
+			state.write_u8(2);
+			hsla.to_f32_array()
+				.iter()
+				.for_each(|v| state.write_u32(v.to_bits()));
+		}
+		Color::Hsva(hsva) => {
+			state.write_u8(3);
+			hsva.to_f32_array()
+				.iter()
+				.for_each(|v| state.write_u32(v.to_bits()));
+		}
+		Color::Hwba(hwba) => {
+			state.write_u8(4);
+			hwba.to_f32_array()
+				.iter()
+				.for_each(|v| state.write_u32(v.to_bits()));
+		}
+		Color::Laba(laba) => {
+			state.write_u8(5);
+			laba.to_f32_array()
+				.iter()
+				.for_each(|v| state.write_u32(v.to_bits()));
+		}
+		Color::Lcha(lcha) => {
+			state.write_u8(6);
+			lcha.to_f32_array()
+				.iter()
+				.for_each(|v| state.write_u32(v.to_bits()));
+		}
+		Color::Oklaba(oklaba) => {
+			state.write_u8(7);
+			oklaba
+				.to_f32_array()
+				.iter()
+				.for_each(|v| state.write_u32(v.to_bits()));
+		}
+		Color::Oklcha(oklcha) => {
+			state.write_u8(8);
+			oklcha
+				.to_f32_array()
+				.iter()
+				.for_each(|v| state.write_u32(v.to_bits()));
+		}
+		Color::Xyza(xyza) => {
+			state.write_u8(9);
+			xyza.to_f32_array()
+				.iter()
+				.for_each(|v| state.write_u32(v.to_bits()));
+		}
+	}
+}
+static MODEL_REGISTRY: Registry<Model> = Registry::new();
+
+impl MaterialParameter {
+	fn apply_to_material(
+		&self,
+		client: &Client,
+		mat: &mut BevyMaterial,
+		parameter_name: &str,
+		asset_server: &AssetServer,
+	) {
+		match self {
+			MaterialParameter::Bool(val) => match parameter_name {
+				"double_sided" => mat.double_sided = *val,
+				v => {
+					error!("unknown param_name ({v}) for color")
+				}
+			},
+			MaterialParameter::Int(_val) => {
+				// nothing uses an int
+			}
+			MaterialParameter::UInt(_val) => {
+				// nothing uses an uint
+			}
+			MaterialParameter::Float(val) => {
+				match parameter_name {
+					"metallic" => mat.metallic = *val,
+					"roughness" => mat.perceptual_roughness = *val,
+					// we probably don't want to expose tex_scale
+					// "tex_scale" => mat.tex_scale = *val,
+					v => {
+						error!("unknown param_name ({v}) for float")
+					}
+				}
+			}
+			MaterialParameter::Vec2(_val) => {
+				// nothing uses a Vec2
+			}
+			MaterialParameter::Vec3(_val) => {
+				// nothing uses a Vec3
+			}
+			MaterialParameter::Color(color) => match parameter_name {
+				"color" => mat.base_color = color.to_bevy(),
+				"emission_factor" => mat.emissive = color.to_bevy().to_linear(),
+				v => {
+					error!("unknown param_name ({v}) for color")
+				}
+			},
 			MaterialParameter::Texture(resource) => {
 				let Some(texture_path) =
 					get_resource_file(resource, client, &[OsStr::new("png"), OsStr::new("jpg")])
 				else {
 					return;
 				};
-				if let Ok(tex) = Tex::from_file(texture_path, true, None) {
-					params.set_texture(parameter_name, &tex);
+				let handle = asset_server.load(texture_path);
+				match parameter_name {
+					"diffuse" => mat.base_color_texture = Some(handle),
+					"emission" => mat.emissive_texture = Some(handle),
+					"metal" => mat.metallic_roughness_texture = Some(handle),
+					"occlusion" => mat.occlusion_texture = Some(handle),
+					v => {
+						error!("unknown param_name ({v}) for texture");
 					}
 				}
+				// mat.alpha_mode = AlphaMode::Blend;
+			}
 		}
 	}
 }
 
 pub struct ModelPart {
-	id: i32,
+	entity: OnceLock<EntityHandle>,
+	mesh_entity: OnceLock<EntityHandle>,
 	path: String,
 	space: Arc<Spatial>,
-	model: Weak<Model>,
+	_model: Weak<Model>,
 	pending_material_parameters: Mutex<FxHashMap<String, MaterialParameter>>,
-	pending_material_replacement: Mutex<Option<Arc<MaterialWrapper>>>,
+	pending_material_replacement: Mutex<Option<Handle<BevyMaterial>>>,
+	holdout: AtomicBool,
 	aliases: AliasList,
+	bounds: OnceLock<Aabb>,
 }
 impl ModelPart {
-	fn create_for_model(model: &Arc<Model>, sk_model: &SKModel) {
-		HOLDOUT_MATERIAL.get_or_init(|| {
-			let mut mat = Material::copy(&Material::unlit());
-			mat.transparency(Transparency::None);
-			mat.color_tint(Color128::BLACK_TRANSPARENT);
-			Arc::new(MaterialWrapper(mat))
-		});
-
-		let nodes = sk_model.get_nodes();
-		for part in nodes.all() {
-			ModelPart::create(model, &part);
-		}
-	}
-
-	fn create(model: &Arc<Model>, part: &stereokit_rust::model::ModelNode) -> Option<Arc<Self>> {
-		let mut parts = model.parts.lock();
-		let parent_part = part
-			.get_parent()
-			.and_then(|part| parts.iter().find(|p| p.id == *part.get_id()));
-
-		let stardust_model_part = model.space.node()?;
-		let client = stardust_model_part.get_client()?;
-		let mut part_path = parent_part
-			.map(|n| n.path.clone() + "/")
-			.unwrap_or_default();
-		part_path += part.get_name().unwrap();
-
-		let node = client.scenegraph.add_node(Node::generate(&client, false));
-		let spatial_parent = parent_part
-			.map(|n| n.space.clone())
-			.unwrap_or_else(|| model.space.clone());
-
-		let local_transform = unsafe { part.get_local_transform().m };
-		let space = Spatial::add_to(
-			&node,
-			Some(spatial_parent),
-			Mat4::from_cols_array(&local_transform),
-			false,
-		);
-
-		let _ = space.bounding_box_calc.set(|node| {
-			let Ok(model_part) = node.get_aspect::<ModelPart>() else {
-				return Bounds::default();
-			};
-			let Some(model) = model_part.model.upgrade() else {
-				return Bounds::default();
-			};
-			let Some(sk_model) = model.sk_model.get() else {
-				return Bounds::default();
-			};
-			let model_nodes = sk_model.get_nodes();
-			let Some(model_node) = model_nodes.get_index(model_part.id) else {
-				return Bounds::default();
-			};
-			let Some(sk_mesh) = model_node.get_mesh() else {
-				return Bounds::default();
-			};
-			sk_mesh.get_bounds()
-		});
-
-		let model_part = Arc::new(ModelPart {
-			id: *part.get_id(),
-			path: part_path,
-			space,
-			model: Arc::downgrade(model),
-			pending_material_parameters: Mutex::new(FxHashMap::default()),
-			pending_material_replacement: Mutex::new(None),
-			aliases: AliasList::default(),
-		});
-		<ModelPart as ModelPartAspect>::add_node_members(&node);
-		node.add_aspect_raw(model_part.clone());
-		parts.push(model_part.clone());
-		Some(model_part)
-	}
-
-	pub fn replace_material(&self, replacement: Arc<MaterialWrapper>) {
-		let shared_material = MATERIAL_REGISTRY.add_or_get(replacement);
+	pub fn replace_material(&self, replacement: Handle<BevyMaterial>) {
 		self.pending_material_replacement
 			.lock()
-			.replace(shared_material);
+			.replace(replacement);
 	}
-	/// only to be run on the main thread
-	pub fn replace_material_now(&self, replacement: &Material) {
-		let Some(model) = self.model.upgrade() else {
-			return;
-		};
-		let Some(sk_model) = model.sk_model.get() else {
-			return;
-		};
-		let nodes = sk_model.get_nodes();
-		let Some(mut part) = nodes.get_index(self.id) else {
-			return;
-		};
-		let shared_material =
-			MATERIAL_REGISTRY.add_or_get(Arc::new(MaterialWrapper(replacement.copy())));
-		part.material(&shared_material.0);
+	pub fn set_material_parameter(&self, parameter_name: String, value: MaterialParameter) {
+		self.pending_material_parameters
+			.lock()
+			.insert(parameter_name, value);
 	}
-
-	fn update(&self) {
-		let Some(model) = self.model.upgrade() else {
-			return;
-		};
-		let Some(sk_model) = model.sk_model.get() else {
-			return;
-		};
-		let Some(node) = model.space.node() else {
-			return;
-		};
-		let nodes = sk_model.get_nodes();
-		let Some(mut part) = nodes.get_index(self.id) else {
-			return;
-		};
-		part.model_transform(Spatial::space_to_space_matrix(
-			Some(&self.space),
-			Some(&model.space),
-		));
-
-		let Some(client) = node.get_client() else {
-			return;
-		};
-
-		if let Some(material_replacement) = self.pending_material_replacement.lock().take() {
-			part.material(&material_replacement.0);
-		}
-
-		'mat_params: {
-			let mut material_parameters = self.pending_material_parameters.lock();
-			if !material_parameters.is_empty() {
-				let Some(material) = part.get_material() else {
-					break 'mat_params;
-				};
-				let new_material = material.copy();
-				for (parameter_name, parameter_value) in material_parameters.drain() {
-					parameter_value.apply_to_material(&client, &new_material, &parameter_name);
-				}
-
-				let shared_material =
-					MATERIAL_REGISTRY.add_or_get(Arc::new(MaterialWrapper(new_material)));
-				part.material(&shared_material.0);
-			}
-		}
-	}
-}
-impl Aspect for ModelPart {
-	const NAME: &'static str = "ModelPart";
 }
 impl ModelPartAspect for ModelPart {
 	#[doc = "Set this model part's material to one that cuts a hole in the world. Often used for overlays/passthrough where you want to show the background through an object."]
 	fn apply_holdout_material(node: Arc<Node>, _calling_client: Arc<Client>) -> Result<()> {
 		let model_part = node.get_aspect::<ModelPart>()?;
-		model_part.replace_material(HOLDOUT_MATERIAL.get().unwrap().clone());
+		model_part.holdout.store(true, Ordering::Relaxed);
 		Ok(())
 	}
 
@@ -297,20 +490,41 @@ impl ModelPartAspect for ModelPart {
 		value: MaterialParameter,
 	) -> Result<()> {
 		let model_part = node.get_aspect::<ModelPart>()?;
-		model_part
-			.pending_material_parameters
-			.lock()
-			.insert(parameter_name, value);
-
+		model_part.set_material_parameter(parameter_name, value);
 		Ok(())
 	}
 }
+#[derive(Default, Resource)]
+pub struct MaterialRegistry(FxHashMap<HashedPbrMaterial, Handle<BevyMaterial>>);
+impl MaterialRegistry {
+	/// returns strong handle for PbrMaterial elminitating duplications
+	pub fn get_handle(
+		&mut self,
+		material: BevyMaterial,
+		materials: &mut ResMut<Assets<BevyMaterial>>,
+	) -> Handle<BevyMaterial> {
+		let hash = HashedPbrMaterial::new(&material);
+		match self
+			.0
+			.get(&hash)
+			.and_then(|v| materials.get_strong_handle(v.id()))
+		{
+			Some(v) => v,
+			None => {
+				let handle = materials.add(material);
+				self.0.insert(hash, handle.clone_weak());
+				handle
+			}
+		}
+	}
+}
 
 pub struct Model {
-	space: Arc<Spatial>,
+	spatial: Arc<Spatial>,
 	_resource_id: ResourceID,
-	sk_model: OnceCell<SKModel>,
-	parts: Mutex<Vec<Arc<ModelPart>>>,
+	bevy_scene_entity: OnceLock<EntityHandle>,
+	parts: OnceLock<Vec<Arc<ModelPart>>>,
+	pre_bound_parts: Mutex<Vec<Arc<ModelPart>>>,
 }
 impl Model {
 	pub fn add_to(node: &Arc<Node>, resource_id: ResourceID) -> Result<Arc<Model>> {
@@ -322,47 +536,67 @@ impl Model {
 		.ok_or_else(|| eyre!("Resource not found"))?;
 
 		let model = Arc::new(Model {
-			space: node.get_aspect::<Spatial>().unwrap().clone(),
+			spatial: node.get_aspect::<Spatial>().unwrap().clone(),
 			_resource_id: resource_id,
-			sk_model: OnceCell::new(),
-			parts: Mutex::new(Vec::default()),
+			bevy_scene_entity: OnceLock::new(),
+			pre_bound_parts: Mutex::default(),
+			parts: OnceLock::new(),
 		});
-		<Model as ModelAspect>::add_node_members(node);
+		LOAD_MODEL
+			.send((model.clone(), pending_model_path))
+			.unwrap();
 		MODEL_REGISTRY.add_raw(&model);
 
-		// technically doing this in anything but the main thread isn't a good idea but dangit we need those model nodes ASAP
-		let sk_model = SKModel::copy(SKModel::from_file(
-			pending_model_path.to_str().unwrap(),
-			None,
-		)?);
-		ModelPart::create_for_model(&model, &sk_model);
-		let _ = model.sk_model.set(sk_model);
 		node.add_aspect_raw(model.clone());
 		Ok(model)
 	}
-
-	fn draw(&self, token: &MainThreadToken) {
-		let Some(sk_model) = self.sk_model.get() else {
-			return;
+	pub fn get_model_part(self: &Arc<Self>, part_path: String) -> Result<Arc<ModelPart>> {
+		let part = match self
+			.parts
+			.get()
+			.map(|v| v.iter().find(|p| p.path == part_path))
+		{
+			Some(Some(part)) => part.clone(),
+			Some(None) => {
+				let paths = self
+					.parts
+					.get()
+					.unwrap()
+					.iter()
+					.map(|p| &p.path)
+					.collect::<Vec<_>>();
+				bail!(
+					"Couldn't find model part at path {part_path}, all available paths: {paths:?}",
+				);
+			}
+			None => {
+				// TODO: this could be a denail of service vector
+				let client = self.spatial.node().unwrap().get_client().unwrap();
+				let part_node = client.scenegraph.add_node(Node::generate(&client, false));
+				let spatial = Spatial::add_to(
+					&part_node,
+					Some(self.spatial.clone()),
+					Mat4::IDENTITY,
+					false,
+				);
+				let part = part_node.add_aspect(ModelPart {
+					entity: OnceLock::new(),
+					mesh_entity: OnceLock::new(),
+					path: part_path,
+					space: spatial,
+					_model: Arc::downgrade(self),
+					pending_material_parameters: Mutex::default(),
+					pending_material_replacement: Mutex::default(),
+					holdout: AtomicBool::new(false),
+					aliases: AliasList::default(),
+					bounds: OnceLock::new(),
+				});
+				self.pre_bound_parts.lock().push(part.clone());
+				part
+			}
 		};
-		let parts = self.parts.lock();
-		for model_node in &*parts {
-			model_node.update();
+		Ok(part)
 	}
-		drop(parts);
-
-		if let Some(node) = self.space.node() {
-			if node.enabled() {
-				sk_model.draw(token, self.space.global_transform(), None, None);
-			}
-		}
-	}
-}
-// TODO: proper hread safety in stereokit_rust (probably just bind stereokit directly)
-unsafe impl Send for Model {}
-unsafe impl Sync for Model {}
-impl Aspect for Model {
-	const NAME: &'static str = "Model";
 }
 impl ModelAspect for Model {
 	#[doc = "Bind a model part to the node with the ID input."]
@@ -371,13 +605,9 @@ impl ModelAspect for Model {
 		calling_client: Arc<Client>,
 		id: u64,
 		part_path: String,
-	) -> color_eyre::eyre::Result<()> {
+	) -> Result<()> {
 		let model = node.get_aspect::<Model>()?;
-		let parts = model.parts.lock();
-		let Some(part) = parts.iter().find(|p| p.path == part_path) else {
-			let paths = parts.iter().map(|p| &p.path).collect::<Vec<_>>();
-			bail!("Couldn't find model part at path {part_path}, all available paths: {paths:?}",)
-		};
+		let part = model.get_model_part(part_path)?;
 		Alias::create_with_id(
 			&part.space.node().unwrap(),
 			&calling_client,
@@ -393,9 +623,3 @@ impl Drop for Model {
 		MODEL_REGISTRY.remove(self);
 	}
 }
-
-pub fn draw_all(token: &MainThreadToken) {
-	for model in MODEL_REGISTRY.get_valid_contents() {
-		model.draw(token);
-	}
-}

src/nodes/drawable/shader_manipulation.rs

@@ -1,137 +0,0 @@
-#![allow(dead_code)]
-use smithay::backend::renderer::gles::{
-	ffi::{self, Gles2, FRAGMENT_SHADER, VERTEX_SHADER},
-	GlesError,
-};
-use stereokit_rust::shader::{Shader, _ShaderT};
-use tracing::error;
-
-struct FfiAssetHeader {
-	asset_type: i32,
-	asset_state: i32,
-	id: u64,
-	index: u64,
-	refs: i32,
-	debug: *mut u8,
-}
-
-struct FfiSkgShader {
-	meta: *mut u8,
-	vertex: u32,
-	pixel: u32,
-	program: u32,
-	compute: u32,
-}
-
-struct FfiShader {
-	header: FfiAssetHeader,
-	shader: FfiSkgShader,
-}
-
-unsafe fn compile_shader(
-	gl: &ffi::Gles2,
-	variant: ffi::types::GLuint,
-	src: &str,
-) -> Result<ffi::types::GLuint, GlesError> {
-	let shader = gl.CreateShader(variant);
-	if shader == 0 {
-		return Err(GlesError::CreateShaderObject);
-	}
-
-	gl.ShaderSource(
-		shader,
-		1,
-		&src.as_ptr() as *const *const u8 as *const *const ffi::types::GLchar,
-		&(src.len() as i32) as *const _,
-	);
-	gl.CompileShader(shader);
-
-	let mut status = ffi::FALSE as i32;
-	gl.GetShaderiv(shader, ffi::COMPILE_STATUS, &mut status as *mut _);
-	if status == ffi::FALSE as i32 {
-		let mut max_len = 0;
-		gl.GetShaderiv(shader, ffi::INFO_LOG_LENGTH, &mut max_len as *mut _);
-
-		let mut error = Vec::with_capacity(max_len as usize);
-		let mut len = 0;
-		gl.GetShaderInfoLog(
-			shader,
-			max_len as _,
-			&mut len as *mut _,
-			error.as_mut_ptr() as *mut _,
-		);
-		error.set_len(len as usize);
-
-		error!(
-			"[GL] {}",
-			std::str::from_utf8(&error).unwrap_or("<Error Message no utf8>")
-		);
-
-		gl.DeleteShader(shader);
-		return Err(GlesError::ShaderCompileError);
-	}
-
-	Ok(shader)
-}
-
-unsafe fn link_program(
-	gl: &ffi::Gles2,
-	vert: ffi::types::GLuint,
-	frag: ffi::types::GLuint,
-) -> Result<ffi::types::GLuint, GlesError> {
-	let program = gl.CreateProgram();
-	gl.AttachShader(program, vert);
-	gl.AttachShader(program, frag);
-	gl.LinkProgram(program);
-	// gl.DetachShader(program, vert);
-	// gl.DetachShader(program, frag);
-	// gl.DeleteShader(vert);
-	// gl.DeleteShader(frag);
-
-	let mut status = ffi::FALSE as i32;
-	gl.GetProgramiv(program, ffi::LINK_STATUS, &mut status as *mut _);
-	if status == ffi::FALSE as i32 {
-		let mut max_len = 0;
-		gl.GetProgramiv(program, ffi::INFO_LOG_LENGTH, &mut max_len as *mut _);
-
-		let mut error = Vec::with_capacity(max_len as usize);
-		let mut len = 0;
-		gl.GetProgramInfoLog(
-			program,
-			max_len as _,
-			&mut len as *mut _,
-			error.as_mut_ptr() as *mut _,
-		);
-		error.set_len(len as usize);
-
-		error!(
-			"[GL] {}",
-			std::str::from_utf8(&error).unwrap_or("<Error Message no utf8>")
-		);
-
-		gl.DeleteProgram(program);
-		return Err(GlesError::ProgramLinkError);
-	}
-
-	Ok(program)
-}
-
-pub unsafe fn shader_inject(
-	c: &Gles2,
-	sk_shader: &mut Shader,
-	vert_str: &str,
-	frag_str: &str,
-) -> Result<(), GlesError> {
-	let gl_vert = compile_shader(c, VERTEX_SHADER, vert_str)?;
-	let gl_frag = compile_shader(c, FRAGMENT_SHADER, frag_str)?;
-	let gl_prog = link_program(c, gl_vert, gl_frag)?;
-
-	let shader = sk_shader.0.as_mut() as *mut _ShaderT as *mut FfiShader;
-	if let Some(shader) = shader.as_mut() {
-		shader.shader.vertex = gl_vert;
-		shader.shader.pixel = gl_frag;
-		shader.shader.program = gl_prog;
-	}
-
-	Ok(())
-}

src/nodes/drawable/shaders.rs

@@ -1,5 +0,0 @@
-// Simula shader with fancy lanzcos sampling
-pub const UNLIT_SHADER_BYTES: &[u8] = include_bytes!("assets/shaders/shader_unlit_gamma.hlsl.sks");
-
-// Simula shader with fancy lanzcos sampling
-pub const PANEL_SHADER_BYTES: &[u8] = include_bytes!("assets/shaders/shader_unlit_simula.hlsl.sks");

src/nodes/drawable/shaders_src/shader_unlit_gamma.hlsl

@@ -1,39 +0,0 @@
-#include "stereokit.hlsli"
-
-//--name = sk/unlit
-//--diffuse     = white
-//--uv_offset   = 0.0, 0.0
-//--uv_scale    = 1.0, 1.0
-Texture2D    diffuse   : register(t0);
-SamplerState diffuse_s : register(s0);
-float2       uv_scale;
-float2       uv_offset;
-
-struct vsIn {
-	float4 pos  : SV_Position;
-	float3 norm : NORMAL0;
-	float2 uv   : TEXCOORD0;
-};
-struct psIn {
-	float4 pos   : SV_POSITION;
-	float2 uv    : TEXCOORD0;
-	uint view_id : SV_RenderTargetArrayIndex;
-};
-
-psIn vs(vsIn input, uint id : SV_InstanceID) {
-	psIn o;
-	o.view_id = id % sk_view_count;
-	id        = id / sk_view_count;
-
-	float3 world = mul(float4(input.pos.xyz, 1), sk_inst[id].world).xyz;
-	o.pos        = mul(float4(world,         1), sk_viewproj[o.view_id]);
-
-	o.uv    = (input.uv + uv_offset) * uv_scale;
-	return o;
-}
-float4 ps(psIn input) : SV_TARGET {
-	float4 col = diffuse.Sample(diffuse_s, input.uv);
-	col.rgb = pow(col.rgb, float3(2.2));
-
-	return col; 
-}

src/nodes/drawable/shaders_src/shader_unlit_simula.hlsl

@@ -1,119 +0,0 @@
-#include "stereokit.hlsli"
-
-// Port of https://github.com/SimulaVR/Simula/blob/master/addons/godot-haskell-plugin/TextShader.tres to StereoKit and HLSL.
-
-//--name = stardust/text_shader
-//--diffuse     = white
-//--uv_offset   = 0.0, 0.0
-//--uv_scale    = 1.0, 1.0
-//--fcFactor    = 1.0
-//--ripple      = 4.0
-//--alpha_min   = 0.0
-//--alpha_max   = 1.0
-Texture2D    diffuse   : register(t0);
-SamplerState diffuse_s : register(s0);
-float4       diffuse_i;
-float2       uv_scale;
-float2       uv_offset;
-float        fcFactor;
-float        ripple;
-float        alpha_min;
-float        alpha_max;
-
-struct vsIn {
-	float4 pos  : SV_Position;
-	float3 norm : NORMAL0;
-	float2 uv   : TEXCOORD0;
-};
-struct psIn {
-	float4 pos   : SV_POSITION;
-	float2 uv    : TEXCOORD0;
-	uint view_id : SV_RenderTargetArrayIndex;
-};
-
-psIn vs(vsIn input, uint id : SV_InstanceID) {
-	psIn o;
-	o.view_id = id % sk_view_count;
-	id        = id / sk_view_count;
-
-	float3 world = mul(float4(input.pos.xyz, 1), sk_inst[id].world).xyz;
-	o.pos        = mul(float4(world,         1), sk_viewproj[o.view_id]);
-
-	o.uv    = (input.uv + uv_offset) * uv_scale;
-	return o;
-}
-
-float map(float value, float min1, float max1, float min2, float max2) {
-  return min2 + (value - min1) * (max2 - min2) / (max1 - min1);
-}
-
-// float gaussian(float x, float t) {
-// 	float PI = 3.14159265358;
-// 	return exp(-x*x/(2.0 * t*t))/(sqrt(2.0*PI)*t);
-// }
-
-float besselI0(float x) {
-	return 1.0 + pow(x, 2.0) * (0.25 +  pow(x, 2.0) * (0.015625 +  pow(x, 2.0) * (0.000434028 +  pow(x, 2.0) * (6.78168e-6 +  pow(x, 2.0) * (6.78168e-8 +  pow(x, 2.0) * (4.7095e-10 +  pow(x, 2.0) * (2.40281e-12 + pow(x, 2.0) * (9.38597e-15 + pow(x, 2.0) * (2.8969e-17 + 7.24226e-20 * pow(x, 2.0))))))))));
-}
-
-float kaiser(float x, float alpha) {
-	if (x > 1.0) { 
-		return 0.0;
-	}
-	return besselI0(alpha * sqrt(1.0-x*x));
-}
-
-float4 lowpassFilter(Texture2D tex, sampler2D texSampler, float2 uv, float alpha) {
-	float PI = 3.14159265358;
-	
-	float4 q = float4(0.0);
-	
-	float2 dx_uv = ddx(uv);
-	float2 dy_uv = ddy(uv);
-	//float width = sqrt(max(dot(dx_uv, dx_uv), dot(dy_uv, dy_uv)));
-	float2 width = abs(float2(dx_uv.x, dy_uv.y));
-	
-	float2 pixelWidth = floor(width * diffuse_i.xy);
-	float2 aspectRatio = normalize(pixelWidth);
-	
-	float2 xyf = uv * diffuse_i.xy;
-	int2 xy = int2(xyf);
-	
-	pixelWidth = clamp(pixelWidth, float2(1.0), float2(2.0));
-
-	int2 start = xy - int2(pixelWidth);
-	int2 end = xy + int2(pixelWidth);
-	
-	float4 outColor = float4(0.0);
-	
-	float qSum = 0.0;
-	
-	for (int v = start.y; v <= end.y; v++) {
-		for (int u = start.x; u <= end.x; u++) {
-			float kx = fcFactor * (xyf.x - float(u))/pixelWidth.x;
-			float ky = fcFactor * (xyf.y - float(v))/pixelWidth.y;
-			 
-			//float lanczosValue = gaussian(kx, fcx);
-			float lanczosValue = kaiser(sqrt(kx*kx + ky*ky), alpha);
-			
-			q += tex.Sample(texSampler, (float2(u, v)+float2(0.5))/diffuse_i.xy) * lanczosValue;
-			// q += tex.Load(int3(u, v, 0)) * lanczosValue;
-			qSum += lanczosValue;
-		}
-	}
-	
-	return q/qSum;
-}
-
-float4 ps(psIn input) : SV_TARGET {
-	float gamma = 2.2;
-	// float4 col = diffuse.Sample(diffuse_s, input.uv);
-
-	// float4 col = lowpassFilter(diffuse, diffuse_s, diffuse_i.xy, float2(1.0 - input.uv.x, input.uv.y), ripple);
-	float4 col = lowpassFilter(diffuse, diffuse_s, input.uv, ripple);
-	// float4 col = diffuse.Sample(diffuse_s, input.uv);
-	col.rgb = pow(col.rgb, float3(gamma));
-	col.a = map(col.a, 0, 1, alpha_min, alpha_max);
-
-	return col; 
-}

src/nodes/drawable/text.rs

@@ -1,42 +1,198 @@
 use crate::{
-	core::{client::Client, destroy_queue, registry::Registry, resource::get_resource_file},
-	nodes::{spatial::Spatial, Aspect, Node},
+	BevyMaterial,
+	core::{
+		bevy_channel::{BevyChannel, BevyChannelReader},
+		client::Client,
+		color::ColorConvert,
+		entity_handle::EntityHandle,
+		error::Result,
+		registry::Registry,
+		resource::get_resource_file,
+	},
+	nodes::{
+		Node,
+		drawable::XAlign,
+		spatial::{Spatial, SpatialNode},
+	},
 };
-use color_eyre::eyre::{eyre, Result};
-use glam::{vec3, Mat4, Vec2};
-use once_cell::sync::OnceCell;
+use bevy::{platform::collections::HashMap, prelude::*, render::mesh::MeshAabb};
+use bevy_mesh_text_3d::{
+	Align, Attrs, MeshTextPlugin, Settings as FontSettings, generate_meshes,
+	text_glyphs::TextGlyphs,
+};
+use color_eyre::eyre::eyre;
+use core::f32;
+use cosmic_text::Metrics;
 use parking_lot::Mutex;
-use std::{ffi::OsStr, path::PathBuf, sync::Arc};
-use stereokit_rust::{
-	font::Font,
-	sk::MainThreadToken,
-	system::{TextAlign, TextFit, TextStyle as SkTextStyle},
-	util::{Color128, Color32},
-};
+use std::{ffi::OsStr, mem, path::PathBuf, sync::Arc};
 
-use super::{TextAspect, TextStyle};
+static SPAWN_TEXT: BevyChannel<Arc<Text>> = BevyChannel::new();
+
+pub struct TextNodePlugin;
+
+impl Plugin for TextNodePlugin {
+	fn build(&self, app: &mut App) {
+		// Text init stuff
+		// 1.0 for font size in meters
+		app.add_plugins(MeshTextPlugin::new(1.0));
+		app.world_mut()
+			.resource_mut::<FontSettings>()
+			.font_system
+			.db_mut()
+			.load_system_fonts();
+
+		SPAWN_TEXT.init(app);
+		app.init_resource::<MaterialRegistry>();
+		app.add_systems(Update, spawn_text);
+	}
+}
+
+fn spawn_text(
+	mut mpsc: ResMut<BevyChannelReader<Arc<Text>>>,
+	mut cmds: Commands,
+	mut font_settings: ResMut<FontSettings>,
+	mut material_registry: ResMut<MaterialRegistry>,
+	mut materials: ResMut<Assets<BevyMaterial>>,
+	mut meshes: ResMut<Assets<Mesh>>,
+	mut font_registry: Local<FontDatabaseRegistry>,
+) {
+	while let Some(text) = mpsc.read() {
+		if let Some(entity) = text.entity.lock().take() {
+			cmds.entity(*entity).despawn();
+		}
+		let style = text.data.lock();
+		let old_db = text.font_path.clone().map(|p| {
+			let db = font_registry.get(p);
+			mem::swap(font_settings.font_system.db_mut(), db);
+			db
+		});
+		let attrs = Attrs::new().weight(cosmic_text::Weight::BOLD);
+		let alignment = Some(match style.text_align_x {
+			super::XAlign::Left => Align::Right,
+			super::XAlign::Center => Align::Center,
+			super::XAlign::Right => Align::Left,
+		});
+		let text_string = text.text.lock().clone();
+		let mut text_glyphs = TextGlyphs::new(
+			Metrics {
+				font_size: style.character_height,
+				line_height: style.character_height,
+			},
+			[(text_string.as_str(), attrs.clone())],
+			&attrs,
+			&mut font_settings.font_system,
+			alignment,
+		);
+		let max_width = style.bounds.as_ref().map(|v| v.bounds.x);
+		let max_height = style.bounds.as_ref().map(|v| v.bounds.x);
+		let (width, _height) =
+			text_glyphs.measure(max_width, max_height, &mut font_settings.font_system);
+		let char_meshes = generate_meshes(
+			bevy_mesh_text_3d::InputText::Simple {
+				text: text_string,
+				material: material_registry.get_handle(
+					BevyMaterial {
+						base_color: style.color.to_bevy(),
+						emissive: Color::WHITE.to_linear(),
+						metallic: 0.0,
+						perceptual_roughness: 1.0,
+						// If alpha is supported on text we need to change this
+						alpha_mode: AlphaMode::Opaque,
+						double_sided: false,
+						..default()
+					},
+					&mut materials,
+				),
+				attrs,
+			},
+			&mut font_settings,
+			bevy_mesh_text_3d::Parameters {
+				extrusion_depth: 0.0,
+				font_size: style.character_height,
+				line_height: style.character_height,
+				alignment,
+				max_width,
+				max_height,
+			},
+			&mut meshes,
+		);
+		if let Some(db) = old_db {
+			mem::swap(font_settings.font_system.db_mut(), db);
+		}
+		let Ok(char_meshes) =
+			char_meshes.inspect_err(|err| error!("unable to create text meshes: {err}"))
+		else {
+			continue;
+		};
+		let dist = char_meshes.iter().fold(f32::MAX, |dist, v| {
+			dist.min(
+				v.transform.translation.x
+					- meshes
+						.get(&v.mesh)
+						.unwrap()
+						.compute_aabb()
+						.unwrap_or_default()
+						.half_extents
+						.x,
+			)
+		});
+		// TODO: text align
+		let letters = char_meshes
+			.into_iter()
+			.map(|v| {
+				cmds.spawn((
+					Mesh3d(v.mesh),
+					MeshMaterial3d(v.material),
+					// rotation is sus, might be related to the gltf coordinate system
+					Transform::from_rotation(Quat::from_rotation_y(f32::consts::PI))
+						* Transform::from_xyz(
+							-dist
+								+ match style.bounds.as_ref().map(|v| v.anchor_align_x) {
+									Some(XAlign::Center) => width * -0.5,
+									Some(XAlign::Right) => -width,
+									Some(XAlign::Left) => 0.0,
+									None => 0.0,
+								},
+							0.0,
+							0.0,
+						) * v.transform,
+				))
+				.id()
+			})
+			.collect::<Vec<_>>();
+		let entity = cmds
+			.spawn((
+				Name::new("TextNode"),
+				SpatialNode(Arc::downgrade(&text.spatial)),
+			))
+			.add_children(&letters)
+			.id();
+		text.entity.lock().replace(EntityHandle(entity));
+	}
+}
+
+#[derive(Default)]
+struct FontDatabaseRegistry(HashMap<PathBuf, cosmic_text::fontdb::Database>);
+impl FontDatabaseRegistry {
+	fn get(&mut self, path: PathBuf) -> &mut cosmic_text::fontdb::Database {
+		self.0.entry(path).or_insert_with_key(|path| {
+			let mut db = cosmic_text::fontdb::Database::new();
+			if let Err(err) = db.load_font_file(path) {
+				error!("unable to load font file {} {err}", path.to_string_lossy());
+			};
+			db
+		})
+	}
+}
+
+use super::{TextAspect, TextStyle, model::MaterialRegistry};
 
 static TEXT_REGISTRY: Registry<Text> = Registry::new();
 
-fn convert_align(x_align: super::XAlign, y_align: super::YAlign) -> TextAlign {
-	match (x_align, y_align) {
-		(super::XAlign::Left, super::YAlign::Top) => TextAlign::TopLeft,
-		(super::XAlign::Left, super::YAlign::Center) => TextAlign::CenterLeft,
-		(super::XAlign::Left, super::YAlign::Bottom) => TextAlign::BottomLeft,
-		(super::XAlign::Center, super::YAlign::Top) => TextAlign::Center,
-		(super::XAlign::Center, super::YAlign::Center) => TextAlign::Center,
-		(super::XAlign::Center, super::YAlign::Bottom) => TextAlign::BottomCenter,
-		(super::XAlign::Right, super::YAlign::Top) => TextAlign::TopRight,
-		(super::XAlign::Right, super::YAlign::Center) => TextAlign::CenterRight,
-		(super::XAlign::Right, super::YAlign::Bottom) => TextAlign::BottomRight,
-	}
-}
-
 pub struct Text {
-	space: Arc<Spatial>,
+	spatial: Arc<Spatial>,
 	font_path: Option<PathBuf>,
-	style: OnceCell<SkTextStyle>,
-
+	entity: Mutex<Option<EntityHandle>>,
 	text: Mutex<String>,
 	data: Mutex<TextStyle>,
 }
@@ -44,96 +200,20 @@ impl Text {
 	pub fn add_to(node: &Arc<Node>, text: String, style: TextStyle) -> Result<Arc<Text>> {
 		let client = node.get_client().ok_or_else(|| eyre!("Client not found"))?;
 		let text = TEXT_REGISTRY.add(Text {
-			space: node.get_aspect::<Spatial>().unwrap().clone(),
+			spatial: node.get_aspect::<Spatial>().unwrap().clone(),
 			font_path: style.font.as_ref().and_then(|res| {
 				get_resource_file(res, &client, &[OsStr::new("ttf"), OsStr::new("otf")])
 			}),
-			style: OnceCell::new(),
 
+			entity: Mutex::new(None),
 			text: Mutex::new(text),
 			data: Mutex::new(style),
 		});
-		<Text as TextAspect>::add_node_members(node);
 		node.add_aspect_raw(text.clone());
+		_ = SPAWN_TEXT.send(text.clone());
 
 		Ok(text)
 	}
-
-	fn draw(&self, token: &MainThreadToken) {
-		let style =
-			self.style
-				.get_or_try_init(|| -> Result<SkTextStyle, color_eyre::eyre::Error> {
-					let font = self
-						.font_path
-						.as_deref()
-						.and_then(|path| Font::from_file(path).ok())
-						.unwrap_or_default();
-					Ok(SkTextStyle::from_font(font, 1.0, Color32::WHITE))
-				});
-
-		if let Ok(style) = style {
-			let text = self.text.lock();
-			let data = self.data.lock();
-			let transform = self.space.global_transform()
-				* Mat4::from_scale(vec3(
-					data.character_height,
-					data.character_height,
-					data.character_height,
-				));
-			if let Some(bounds) = &data.bounds {
-				stereokit_rust::system::Text::add_in(
-					token,
-					&*text,
-					transform,
-					Vec2::from(bounds.bounds) / data.character_height,
-					match bounds.fit {
-						super::TextFit::Wrap => TextFit::Wrap,
-						super::TextFit::Clip => TextFit::Clip,
-						super::TextFit::Squeeze => TextFit::Squeeze,
-						super::TextFit::Exact => TextFit::Exact,
-						super::TextFit::Overflow => TextFit::Overflow,
-					},
-					Some(*style),
-					Some(Color128::new(
-						data.color.c.r,
-						data.color.c.g,
-						data.color.c.b,
-						data.color.a,
-					)),
-					data.bounds
-						.as_ref()
-						.map(|b| convert_align(b.anchor_align_x, b.anchor_align_y)),
-					Some(convert_align(data.text_align_x, data.text_align_y)),
-					None,
-					None,
-					None,
-				);
-			} else {
-				stereokit_rust::system::Text::add_at(
-					token,
-					&*text,
-					transform,
-					Some(*style),
-					Some(Color128::new(
-						data.color.c.r,
-						data.color.c.g,
-						data.color.c.b,
-						data.color.a,
-					)),
-					data.bounds
-						.as_ref()
-						.map(|b| convert_align(b.anchor_align_x, b.anchor_align_y)),
-					Some(convert_align(data.text_align_x, data.text_align_y)),
-					None,
-					None,
-					None,
-				);
-			}
-		}
-	}
-}
-impl Aspect for Text {
-	const NAME: &'static str = "Text";
 }
 impl TextAspect for Text {
 	fn set_character_height(
@@ -143,30 +223,19 @@ impl TextAspect for Text {
 	) -> Result<()> {
 		let this_text = node.get_aspect::<Text>()?;
 		this_text.data.lock().character_height = height;
+		_ = SPAWN_TEXT.send(this_text);
 		Ok(())
 	}
 
 	fn set_text(node: Arc<Node>, _calling_client: Arc<Client>, text: String) -> Result<()> {
 		let this_text = node.get_aspect::<Text>()?;
 		*this_text.text.lock() = text;
+		_ = SPAWN_TEXT.send(this_text);
 		Ok(())
 	}
 }
 impl Drop for Text {
 	fn drop(&mut self) {
-		if let Some(style) = self.style.take() {
-			destroy_queue::add(style);
-		}
 		TEXT_REGISTRY.remove(self);
 	}
 }
-
-pub fn draw_all(token: &MainThreadToken) {
-	for text in TEXT_REGISTRY.get_valid_contents() {
-		if let Some(node) = text.space.node() {
-			if node.enabled() {
-				text.draw(token);
-			}
-		}
-	}
-}

src/nodes/fields.rs

@@ -1,25 +1,36 @@
 use super::alias::{Alias, AliasInfo};
 use super::spatial::{
-	Spatial, SPATIAL_REF_GET_LOCAL_BOUNDING_BOX_SERVER_OPCODE,
+	SPATIAL_REF_GET_LOCAL_BOUNDING_BOX_SERVER_OPCODE,
 	SPATIAL_REF_GET_RELATIVE_BOUNDING_BOX_SERVER_OPCODE, SPATIAL_REF_GET_TRANSFORM_SERVER_OPCODE,
+	Spatial,
 };
-use super::{Aspect, Node};
+use super::{Aspect, AspectIdentifier, Node};
+use crate::DbusConnection;
 use crate::core::client::Client;
-use crate::create_interface;
-use crate::nodes::spatial::Transform;
+use crate::core::error::Result;
+use crate::core::registry::Registry;
 use crate::nodes::spatial::SPATIAL_ASPECT_ALIAS_INFO;
 use crate::nodes::spatial::SPATIAL_REF_ASPECT_ALIAS_INFO;
-use color_eyre::eyre::{OptionExt, Result};
-use glam::{vec2, vec3, vec3a, Vec3, Vec3A, Vec3Swizzles};
-use once_cell::sync::Lazy;
+use crate::nodes::spatial::Transform;
+use bevy::app::{Plugin, Update};
+use bevy::color::Color;
+use bevy::ecs::resource::Resource;
+use bevy::ecs::schedule::IntoScheduleConfigs;
+use bevy::ecs::system::Res;
+use bevy::gizmos::gizmos::Gizmos;
+use bevy::gizmos::primitives::dim3::GizmoPrimitive3d;
+use bevy::math::primitives::{Cylinder, Torus};
+use color_eyre::eyre::OptionExt;
+use glam::{Vec3, Vec3A, Vec3Swizzles, vec2, vec3, vec3a};
 use parking_lot::Mutex;
 use rustc_hash::FxHashMap;
 use stardust_xr::values::Vector3;
-use std::sync::Arc;
+use std::sync::{Arc, LazyLock, Weak};
+use zbus::interface;
 
 // TODO: get SDFs working properly with non-uniform scale and so on, output distance relative to the spatial it's compared against
 
-pub static FIELD_ALIAS_INFO: Lazy<AliasInfo> = Lazy::new(|| AliasInfo {
+pub static FIELD_ALIAS_INFO: LazyLock<AliasInfo> = LazyLock::new(|| AliasInfo {
 	server_methods: vec![
 		SPATIAL_REF_GET_TRANSFORM_SERVER_OPCODE,
 		SPATIAL_REF_GET_LOCAL_BOUNDING_BOX_SERVER_OPCODE,
@@ -32,10 +43,99 @@ pub static FIELD_ALIAS_INFO: Lazy<AliasInfo> = Lazy::new(|| AliasInfo {
 	..Default::default()
 });
 
+pub struct FieldDebugGizmoPlugin;
+impl Plugin for FieldDebugGizmoPlugin {
+	fn build(&self, app: &mut bevy::app::App) {
+		let (tx, rx) = tokio::sync::watch::channel(false);
+		let conn = app.world().resource::<DbusConnection>().0.clone();
+		tokio::spawn(async move {
+			_ = conn
+				.object_server()
+				.at("/org/stardustxr/Server", FieldDebugGizmos { state: tx })
+				.await;
+		});
+		app.insert_resource(FieldDebugGizmosEnabled(rx));
+		app.add_systems(
+			Update,
+			draw_field_gizmos.run_if(|res: Res<FieldDebugGizmosEnabled>| *res.0.borrow()),
+		);
+	}
+}
+
+#[derive(Resource)]
+struct FieldDebugGizmosEnabled(tokio::sync::watch::Receiver<bool>);
+
+fn draw_field_gizmos(mut gizmos: Gizmos) {
+	FIELD_REGISTRY_DEBUG_GIZMOS
+		.get_valid_contents()
+		.iter()
+		.for_each(|f| {
+			let transform =
+				bevy::transform::components::Transform::from_matrix(f.spatial.global_transform());
+			let color = Color::srgb_u8(0x04, 0xFD, 0x4C);
+			match f.shape.lock().clone() {
+				Shape::Box(size) => gizmos.cuboid(transform.with_scale(size.into()), color),
+				Shape::Cylinder(CylinderShape { length, radius }) => {
+					gizmos
+						.primitive_3d(
+							&Cylinder {
+								radius,
+								half_height: length * 0.5,
+							},
+							transform.to_isometry(),
+							color,
+						)
+						.resolution(32);
+				}
+				Shape::Sphere(radius) => {
+					gizmos.sphere(transform.to_isometry(), radius, color);
+				}
+				Shape::Torus(TorusShape { radius_a, radius_b }) => {
+					let minor_radius;
+					let major_radius;
+					if radius_a >= radius_b {
+						major_radius = radius_a;
+						minor_radius = radius_b;
+					} else {
+						major_radius = radius_b;
+						minor_radius = radius_a;
+					}
+					gizmos
+						.primitive_3d(
+							&Torus {
+								minor_radius,
+								major_radius,
+							},
+							transform.to_isometry(),
+							color,
+						)
+						.minor_resolution(32)
+						.major_resolution(32);
+				}
+			}
+		});
+}
+
+struct FieldDebugGizmos {
+	state: tokio::sync::watch::Sender<bool>,
+}
+
+#[interface(name = "org.stardustxr.debug.FieldDebugGizmos")]
+impl FieldDebugGizmos {
+	fn enable(&mut self) {
+		_ = self.state.send(true);
+	}
+	fn disable(&mut self) {
+		_ = self.state.send(false);
+	}
+}
+
+static FIELD_REGISTRY_DEBUG_GIZMOS: Registry<Field> = Registry::new();
+
 stardust_xr_server_codegen::codegen_field_protocol!();
 
 lazy_static::lazy_static! {
-	pub static ref EXPORTED_FIELDS: Mutex<FxHashMap<u64, Arc<Node>>> = Mutex::new(FxHashMap::default());
+	pub static ref EXPORTED_FIELDS: Mutex<FxHashMap<u64, Weak<Node>>> = Mutex::new(FxHashMap::default());
 }
 
 pub trait FieldTrait: Send + Sync + 'static {
@@ -145,15 +245,71 @@ impl Field {
 			shape: Mutex::new(shape),
 		};
 		let field = node.add_aspect(field);
-		<Field as FieldRefAspect>::add_node_members(node);
-		<Field as FieldAspect>::add_node_members(node);
+		FIELD_REGISTRY_DEBUG_GIZMOS.add_raw(&field);
+		node.add_aspect(FieldRef);
 		Ok(field)
 	}
 }
-impl Aspect for Field {
-	const NAME: &'static str = "Field";
+impl Drop for Field {
+	fn drop(&mut self) {
+		FIELD_REGISTRY_DEBUG_GIZMOS.remove(self);
 	}
-impl FieldRefAspect for Field {
+}
+impl AspectIdentifier for Field {
+	impl_aspect_for_field_aspect_id! {}
+}
+impl Aspect for Field {
+	impl_aspect_for_field_aspect! {}
+}
+impl FieldAspect for Field {
+	fn set_shape(node: Arc<Node>, _calling_client: Arc<Client>, shape: Shape) -> Result<()> {
+		let field = node.get_aspect::<Field>()?;
+		*field.shape.lock() = shape;
+		Ok(())
+	}
+
+	async fn export_field(node: Arc<Node>, _calling_client: Arc<Client>) -> Result<u64> {
+		let id = rand::random();
+		EXPORTED_FIELDS.lock().insert(id, Arc::downgrade(&node));
+		Ok(id)
+	}
+}
+impl FieldTrait for Field {
+	fn spatial_ref(&self) -> &Spatial {
+		&self.spatial
+	}
+	fn local_distance(&self, p: Vec3A) -> f32 {
+		match self.shape.lock().clone() {
+			Shape::Box(size) => {
+				let q = vec3(
+					p.x.abs() - (size.x * 0.5_f32),
+					p.y.abs() - (size.y * 0.5_f32),
+					p.z.abs() - (size.z * 0.5_f32),
+				);
+				let v = vec3a(q.x.max(0_f32), q.y.max(0_f32), q.z.max(0_f32));
+				v.length() + q.x.max(q.y.max(q.z)).min(0_f32)
+			}
+			Shape::Cylinder(CylinderShape { length, radius }) => {
+				let d = vec2(p.xz().length().abs() - radius, p.y.abs() - (length * 0.5));
+				d.x.max(d.y).min(0.0) + d.max(vec2(0.0, 0.0)).length()
+			}
+			Shape::Sphere(radius) => p.length() - radius,
+			Shape::Torus(TorusShape { radius_a, radius_b }) => {
+				let q = vec2(p.xz().length() - radius_a, p.y);
+				q.length() - radius_b
+			}
+		}
+	}
+}
+
+pub struct FieldRef;
+impl AspectIdentifier for FieldRef {
+	impl_aspect_for_field_ref_aspect_id! {}
+}
+impl Aspect for FieldRef {
+	impl_aspect_for_field_ref_aspect! {}
+}
+impl FieldRefAspect for FieldRef {
 	async fn distance(
 		node: Arc<Node>,
 		_calling_client: Arc<Client>,
@@ -205,68 +361,27 @@ impl FieldRefAspect for Field {
 		}))
 	}
 }
-impl FieldAspect for Field {
-	fn set_shape(node: Arc<Node>, _calling_client: Arc<Client>, shape: Shape) -> Result<()> {
-		let field = node.get_aspect::<Field>()?;
-		*field.shape.lock() = shape;
-		Ok(())
-	}
 
-	async fn export_field(node: Arc<Node>, _calling_client: Arc<Client>) -> Result<u64> {
-		let id = rand::random();
-		EXPORTED_FIELDS.lock().insert(id, node);
-		Ok(id)
-	}
-}
-impl FieldTrait for Field {
-	fn spatial_ref(&self) -> &Spatial {
-		&self.spatial
-	}
-	fn local_distance(&self, p: Vec3A) -> f32 {
-		match self.shape.lock().clone() {
-			Shape::Box(size) => {
-				let q = vec3(
-					p.x.abs() - (size.x * 0.5_f32),
-					p.y.abs() - (size.y * 0.5_f32),
-					p.z.abs() - (size.z * 0.5_f32),
-				);
-				let v = vec3a(q.x.max(0_f32), q.y.max(0_f32), q.z.max(0_f32));
-				v.length() + q.x.max(q.y.max(q.z)).min(0_f32)
-			}
-			Shape::Cylinder(CylinderShape { length, radius }) => {
-				let d = vec2(p.xy().length().abs() - radius, p.z.abs() - (length * 0.5));
-				d.x.max(d.y).min(0.0) + d.max(vec2(0.0, 0.0)).length()
-			}
-			Shape::Sphere(radius) => p.length() - radius,
-			Shape::Torus(TorusShape { radius_a, radius_b }) => {
-				let q = vec2(p.xz().length() - radius_a, p.y);
-				q.length() - radius_b
-			}
-		}
-	}
-}
-
-create_interface!(FieldInterface);
-pub struct FieldInterface;
-impl InterfaceAspect for FieldInterface {
+impl InterfaceAspect for Interface {
 	async fn import_field_ref(
 		_node: Arc<Node>,
 		calling_client: Arc<Client>,
 		uid: u64,
 	) -> Result<Arc<Node>> {
-		EXPORTED_FIELDS
+		Ok(EXPORTED_FIELDS
 			.lock()
 			.get(&uid)
+			.and_then(|s| s.upgrade())
 			.map(|s| {
 				Alias::create(
-					s,
+					&s,
 					&calling_client,
 					FIELD_REF_ASPECT_ALIAS_INFO.clone(),
 					None,
 				)
 				.unwrap()
 			})
-			.ok_or_eyre("Couldn't find spatial with that ID")
+			.ok_or_eyre("Couldn't import field with that ID")?)
 	}
 
 	fn create_field(

src/nodes/input/hand.rs

@@ -1,7 +1,7 @@
 use super::{Finger, Hand, InputDataTrait, InputHandler, InputMethod, Joint, Thumb};
 use crate::nodes::fields::{Field, FieldTrait};
 use crate::nodes::spatial::Spatial;
-use glam::{vec3a, Mat4, Quat};
+use glam::{Mat4, Quat, vec3a};
 use std::sync::Arc;
 
 impl Default for Joint {

src/nodes/input/handler.rs

@@ -1,5 +1,5 @@
-use super::{InputHandlerAspect, INPUT_HANDLER_REGISTRY, INPUT_METHOD_REGISTRY};
-use crate::nodes::{alias::AliasList, fields::Field, spatial::Spatial, Aspect, Node};
+use super::{INPUT_HANDLER_REGISTRY, INPUT_METHOD_REGISTRY, InputHandlerAspect};
+use crate::nodes::{Node, alias::AliasList, fields::Field, spatial::Spatial};
 use color_eyre::eyre::Result;
 use std::sync::Arc;
 
@@ -23,9 +23,6 @@ impl InputHandler {
 		Ok(())
 	}
 }
-impl Aspect for InputHandler {
-	const NAME: &'static str = "InputHandler";
-}
 impl InputHandlerAspect for InputHandler {}
 impl PartialEq for InputHandler {
 	fn eq(&self, other: &Self) -> bool {

src/nodes/input/method.rs

@@ -1,18 +1,22 @@
 use super::{
-	input_method_client, InputData, InputDataTrait, InputDataType, InputHandler, InputMethodAspect,
-	InputMethodRefAspect, INPUT_HANDLER_REGISTRY, INPUT_METHOD_REF_ASPECT_ALIAS_INFO,
-	INPUT_METHOD_REGISTRY,
+	INPUT_HANDLER_REGISTRY, INPUT_METHOD_REF_ASPECT_ALIAS_INFO, INPUT_METHOD_REGISTRY, InputData,
+	InputDataTrait, InputDataType, InputHandler, InputMethodAspect, InputMethodRefAspect,
+	input_method_client,
 };
 use crate::{
-	core::{client::Client, registry::Registry},
+	core::{
+		client::Client,
+		error::{Result, ServerError},
+		registry::Registry,
+	},
 	nodes::{
+		Node,
 		alias::{Alias, AliasList},
-		fields::{Field, FIELD_ALIAS_INFO},
+		fields::{FIELD_ALIAS_INFO, Field},
 		spatial::Spatial,
-		Aspect, Node,
 	},
 };
-use color_eyre::eyre::Result;
+use color_eyre::eyre::eyre;
 use parking_lot::Mutex;
 use stardust_xr::values::Datamap;
 use std::sync::{Arc, Weak};
@@ -25,7 +29,7 @@ pub struct InputMethod {
 	handler_aliases: AliasList,
 	handler_field_aliases: AliasList,
 	pub(super) handler_order: Mutex<Vec<Weak<InputHandler>>>,
-	pub internal_capture_requests: Registry<InputHandler>,
+	pub capture_attempts: Registry<InputHandler>,
 	pub captures: Registry<InputHandler>,
 }
 impl InputMethod {
@@ -42,16 +46,15 @@ impl InputMethod {
 			handler_aliases: AliasList::default(),
 			handler_field_aliases: AliasList::default(),
 			handler_order: Mutex::new(Vec::new()),
-			internal_capture_requests: Registry::new(),
+			capture_attempts: Registry::new(),
 			captures: Registry::new(),
 		};
-		<InputMethod as InputMethodRefAspect>::add_node_members(node);
-		<InputMethod as InputMethodAspect>::add_node_members(node);
 		for handler in INPUT_HANDLER_REGISTRY.get_valid_contents() {
 			method.handle_new_handler(&handler);
 		}
 		let method = INPUT_METHOD_REGISTRY.add(method);
 		node.add_aspect_raw(method.clone());
+		node.add_aspect(InputMethodRef);
 		Ok(method)
 	}
 
@@ -131,6 +134,7 @@ impl InputMethod {
 		self.handler_aliases.remove_aspect(handler);
 		self.handler_field_aliases
 			.remove_aspect(handler.field.as_ref());
+		self.capture_attempts.remove(handler);
 	}
 
 	pub(super) fn serialize(&self, alias_id: u64, handler: &Arc<InputHandler>) -> InputData {
@@ -153,24 +157,23 @@ impl InputMethod {
 			captured: self.captures.get_valid_contents().contains(handler),
 		}
 	}
-}
-impl Aspect for InputMethod {
-	const NAME: &'static str = "InputMethod";
-}
-impl InputMethodRefAspect for InputMethod {
-	#[doc = "Have the input handler that this method reference came from capture the method for the next frame."]
-	fn request_capture(
-		node: Arc<Node>,
-		_calling_client: Arc<Client>,
-		handler: Arc<Node>,
-	) -> Result<()> {
-		let input_method = node.get_aspect::<InputMethod>()?;
-		let input_handler = handler.get_aspect::<InputHandler>()?;
 
-		input_method
-			.internal_capture_requests
-			.add_raw(&input_handler);
-		Ok(())
+	pub(super) fn cull_capture_attempts(&self) {
+		let sent = self
+			.handler_order
+			.lock()
+			.iter()
+			.filter_map(Weak::upgrade)
+			.collect::<Registry<InputHandler>>();
+		self.capture_attempts.retain(|handler| {
+			!handler
+				.spatial
+				.node()
+				.and_then(|n| n.get_client())
+				.map(|c| c.unresponsive())
+				.unwrap_or(false)
+				&& sent.contains(handler)
+		});
 	}
 }
 impl InputMethodAspect for InputMethod {
@@ -231,3 +234,46 @@ impl Drop for InputMethod {
 		INPUT_METHOD_REGISTRY.remove(self);
 	}
 }
+
+pub struct InputMethodRef;
+impl InputMethodRefAspect for InputMethodRef {
+	#[doc = "Try to capture the input method with the given handler. When the handler does not get input from the method, it will be released."]
+	fn try_capture(
+		node: Arc<Node>,
+		_calling_client: Arc<Client>,
+		handler: Arc<Node>,
+	) -> Result<()> {
+		let input_method = node.get_aspect::<InputMethod>()?;
+		let input_handler = handler.get_aspect::<InputHandler>()?;
+
+		input_method.capture_attempts.add_raw(&input_handler);
+
+		let Some(handler_alias) = input_method
+			.handler_aliases
+			.get_from_aspect(&*input_handler)
+		else {
+			return Err(ServerError::Report(eyre!(
+				"Internal: Couldn't get handler alias somehow?"
+			)));
+		};
+		input_method_client::request_capture_handler(&node, handler_alias.get_id())
+	}
+
+	#[doc = "If captured by this handler, release it (e.g. the object is let go of after grabbing)."]
+	fn release(node: Arc<Node>, _calling_client: Arc<Client>, handler: Arc<Node>) -> Result<()> {
+		let input_method = node.get_aspect::<InputMethod>()?;
+		let input_handler = handler.get_aspect::<InputHandler>()?;
+
+		input_method.capture_attempts.remove(&input_handler);
+
+		let Some(handler_alias) = input_method
+			.handler_aliases
+			.get_from_aspect(&*input_handler)
+		else {
+			return Err(ServerError::Report(eyre!(
+				"Internal: Couldn't get handler alias somehow?"
+			)));
+		};
+		input_method_client::release_handler(&node, handler_alias.get_id())
+	}
+}

src/nodes/input/mod.rs

@@ -6,17 +6,21 @@ mod method;
 mod pointer;
 mod tip;
 
+use bevy::tasks::ComputeTaskPool;
+use bevy::tasks::ParallelSlice;
 pub use handler::*;
 pub use method::*;
+use tracing::debug_span;
 
+use super::Aspect;
+use super::AspectIdentifier;
 use super::fields::Field;
 use super::spatial::Spatial;
-use crate::create_interface;
+use crate::core::error::Result;
 use crate::nodes::spatial::SPATIAL_ASPECT_ALIAS_INFO;
 use crate::nodes::spatial::SPATIAL_REF_ASPECT_ALIAS_INFO;
 use crate::{core::client::Client, nodes::Node};
 use crate::{core::registry::Registry, nodes::spatial::Transform};
-use color_eyre::eyre::Result;
 use stardust_xr::values::Datamap;
 use std::sync::Arc;
 
@@ -25,6 +29,25 @@ pub static INPUT_HANDLER_REGISTRY: Registry<InputHandler> = Registry::new();
 
 stardust_xr_server_codegen::codegen_input_protocol!();
 
+impl AspectIdentifier for InputHandler {
+	impl_aspect_for_input_handler_aspect_id! {}
+}
+impl Aspect for InputHandler {
+	impl_aspect_for_input_handler_aspect! {}
+}
+impl AspectIdentifier for InputMethod {
+	impl_aspect_for_input_method_aspect_id! {}
+}
+impl Aspect for InputMethod {
+	impl_aspect_for_input_method_aspect! {}
+}
+impl AspectIdentifier for InputMethodRef {
+	impl_aspect_for_input_method_ref_aspect_id! {}
+}
+impl Aspect for InputMethodRef {
+	impl_aspect_for_input_method_ref_aspect! {}
+}
+
 pub trait InputDataTrait {
 	fn transform(&mut self, method: &InputMethod, handler: &InputHandler);
 	fn distance(&self, space: &Arc<Spatial>, field: &Field) -> f32;
@@ -47,9 +70,7 @@ impl InputDataTrait for InputDataType {
 	}
 }
 
-create_interface!(InputInterface);
-pub struct InputInterface;
-impl InterfaceAspect for InputInterface {
+impl InterfaceAspect for Interface {
 	#[doc = "Create an input method node"]
 	fn create_input_method(
 		_node: Arc<Node>,
@@ -101,7 +122,12 @@ pub fn process_input() {
 			};
 			node.enabled()
 		});
-	for handler in INPUT_HANDLER_REGISTRY.get_valid_contents() {
+	INPUT_HANDLER_REGISTRY
+		.get_valid_contents()
+		.into_iter()
+		.par_splat_map(ComputeTaskPool::get(), None, |_, handlers| {
+			for handler in handlers {
+				let _span = debug_span!("handle input handler").entered();
 				for method_alias in handler.method_aliases.get_aliases() {
 					method_alias.set_enabled(false);
 				}
@@ -118,6 +144,7 @@ pub fn process_input() {
 					}
 				};
 
+				let ser_span = debug_span!("serializing input").entered();
 				let (methods, datas) = methods
 					.clone()
 					// filter out methods without the handler in their handler order
@@ -125,7 +152,7 @@ pub fn process_input() {
 						a.handler_order
 							.lock()
 							.iter()
-					.any(|h| h.ptr_eq(&Arc::downgrade(&handler)))
+							.any(|h| h.ptr_eq(&Arc::downgrade(handler)))
 					})
 					// filter out methods without the proper alias
 					.filter_map(|m| {
@@ -141,12 +168,15 @@ pub fn process_input() {
 						a.set_enabled(true);
 					})
 					// serialize the data
-			.map(|(a, m)| (a.clone(), m.serialize(a.get_id(), &handler)))
+					.map(|(a, m)| (a.clone(), m.serialize(a.get_id(), handler)))
 					.unzip::<_, _, Vec<_>, Vec<_>>();
+				drop(ser_span);
 
+				let _span = debug_span!("client input").entered();
 				let _ = input_handler_client::input(&handler_node, &methods, &datas);
 			}
+		});
 	for method in methods {
-		method.internal_capture_requests.clear();
+		method.cull_capture_attempts();
 	}
 }

src/nodes/input/pointer.rs

@@ -3,7 +3,7 @@ use crate::nodes::{
 	fields::{Field, FieldTrait, Ray, RayMarchResult},
 	spatial::Spatial,
 };
-use glam::{vec3, Mat4, Quat};
+use glam::{Mat4, Quat, vec3};
 use std::sync::{Arc, Weak};
 
 impl Default for Pointer {

src/nodes/items/camera.rs

@@ -1,43 +1,27 @@
-use super::{
-	create_item_acceptor_flex, register_item_ui_flex, Item, ItemAcceptor, ItemInterface, ItemType,
-};
+#![allow(dead_code)]
+use super::{Item, ItemType, create_item_acceptor_flex, register_item_ui_flex};
+use crate::bail;
+use crate::core::error::Result;
+use crate::nodes::Aspect;
+use crate::nodes::AspectIdentifier;
 use crate::nodes::items::ITEM_ACCEPTOR_ASPECT_ALIAS_INFO;
 use crate::nodes::items::ITEM_ASPECT_ALIAS_INFO;
 use crate::{
 	core::{client::Client, registry::Registry, scenegraph::MethodResponseSender},
-	create_interface,
 	nodes::{
-		drawable::{
-			model::{MaterialWrapper, ModelPart},
-			shaders::UNLIT_SHADER_BYTES,
-		},
+		Message, Node,
+		drawable::model::ModelPart,
 		items::TypeInfo,
 		spatial::{Spatial, Transform},
-		Message, Node,
 	},
 };
-use color_eyre::eyre::{bail, eyre, Result};
 use glam::Mat4;
 use lazy_static::lazy_static;
 use mint::{ColumnMatrix4, Vector2};
-use once_cell::sync::OnceCell;
 use parking_lot::Mutex;
 
 use stardust_xr::schemas::flex::{deserialize, serialize};
 use std::sync::Arc;
-use stereokit_rust::{
-	material::{Material, Transparency},
-	shader::Shader,
-	sk::MainThreadToken,
-	system::Renderer,
-	tex::{Tex, TexFormat, TexType},
-	util::Color128,
-};
-use tracing::error;
-
-pub struct TexWrapper(pub Tex);
-unsafe impl Send for TexWrapper {}
-unsafe impl Sync for TexWrapper {}
 
 stardust_xr_server_codegen::codegen_item_camera_protocol!();
 lazy_static! {
@@ -48,6 +32,12 @@ lazy_static! {
 		ui: Default::default(),
 		items: Registry::new(),
 		acceptors: Registry::new(),
+		add_acceptor_aspect: |node| {
+			node.add_aspect(CameraItemAcceptor);
+		},
+		add_ui_aspect: |node| {
+			node.add_aspect(CameraItemUi);
+		},
 		new_acceptor_fn: |node, acceptor, acceptor_field| {
 			let _ = camera_item_ui_client::create_acceptor(node, acceptor, acceptor_field);
 		}
@@ -62,30 +52,23 @@ struct FrameInfo {
 pub struct CameraItem {
 	space: Arc<Spatial>,
 	frame_info: Mutex<FrameInfo>,
-	sk_tex: OnceCell<TexWrapper>,
-	sk_mat: OnceCell<Arc<MaterialWrapper>>,
 	applied_to: Registry<ModelPart>,
 	apply_to: Registry<ModelPart>,
 }
 #[allow(unused)]
 impl CameraItem {
 	pub fn add_to(node: &Arc<Node>, proj_matrix: Mat4, px_size: Vector2<u32>) {
-		Item::add_to(
-			node,
-			&ITEM_TYPE_INFO_CAMERA,
-			ItemType::Camera(CameraItem {
+		let item = Arc::new(CameraItem {
 			space: node.get_aspect::<Spatial>().unwrap().clone(),
 			frame_info: Mutex::new(FrameInfo {
 				proj_matrix,
 				px_size,
 			}),
-				sk_tex: OnceCell::new(),
-				sk_mat: OnceCell::new(),
 			applied_to: Registry::new(),
 			apply_to: Registry::new(),
-			}),
-		);
-		// <CameraItem as CameraItemAspect>::node_methods(node);
+		});
+		Item::add_to(node, &ITEM_TYPE_INFO_CAMERA, ItemType::Camera(item.clone()));
+		node.add_aspect_raw(item);
 	}
 
 	fn frame_flex(
@@ -97,7 +80,7 @@ impl CameraItem {
 		response.wrap_sync(move || {
 			let ItemType::Camera(_camera) = &node.get_aspect::<Item>().unwrap().specialization
 			else {
-				return Err(eyre!("Wrong item type?"));
+				bail!("Wrong item type?");
 			};
 			Ok(serialize(())?.into())
 		});
@@ -109,7 +92,7 @@ impl CameraItem {
 		message: Message,
 	) -> Result<()> {
 		let ItemType::Camera(camera) = &node.get_aspect::<Item>().unwrap().specialization else {
-			bail!("Wrong item type?")
+			bail!("Wrong item type?");
 		};
 		let model_part_node =
 			calling_client.get_node("Model part", deserialize(&message.data).unwrap())?;
@@ -125,67 +108,38 @@ impl CameraItem {
 	pub fn send_acceptor_item_created(&self, node: &Node, item: &Arc<Node>) {
 		let _ = camera_item_acceptor_client::capture_item(node, item);
 	}
-
-	pub fn update(&self, token: &MainThreadToken) {
-		let frame_info = self.frame_info.lock();
-		let sk_tex = self.sk_tex.get_or_init(|| {
-			TexWrapper(Tex::gen_color(
-				Color128::default(),
-				frame_info.px_size.x as i32,
-				frame_info.px_size.y as i32,
-				TexType::Rendertarget,
-				TexFormat::RGBA32Linear,
-			))
-		});
-		let sk_mat = self
-			.sk_mat
-			.get_or_try_init(|| -> Result<Arc<MaterialWrapper>> {
-				let shader = Shader::from_memory(UNLIT_SHADER_BYTES)?;
-				let mut mat = Material::new(&shader, None);
-				mat.get_all_param_info().set_texture("diffuse", &sk_tex.0);
-				mat.transparency(Transparency::Blend);
-				Ok(Arc::new(MaterialWrapper(mat)))
-			});
-		let Ok(sk_mat) = sk_mat else {
-			error!("unable to make camera item stereokit texture");
-			return;
-		};
-		for model_part in self.apply_to.take_valid_contents() {
-			model_part.replace_material(sk_mat.clone())
-		}
-
-		if !self.applied_to.is_empty() {
-			Renderer::render_to(
-				token,
-				&sk_tex.0,
-				self.space.global_transform(),
-				frame_info.proj_matrix,
-				None,
-				None,
-				None,
-			)
 }
+impl AspectIdentifier for CameraItem {
+	impl_aspect_for_camera_item_aspect_id! {}
 }
+impl Aspect for CameraItem {
+	impl_aspect_for_camera_item_aspect! {}
 }
 impl CameraItemAspect for CameraItem {}
 
-impl CameraItemAcceptorAspect for ItemAcceptor {
+pub struct CameraItemUi;
+impl AspectIdentifier for CameraItemUi {
+	impl_aspect_for_camera_item_ui_aspect_id! {}
+}
+impl Aspect for CameraItemUi {
+	impl_aspect_for_camera_item_ui_aspect! {}
+}
+impl CameraItemUiAspect for CameraItemUi {}
+
+pub struct CameraItemAcceptor;
+impl AspectIdentifier for CameraItemAcceptor {
+	impl_aspect_for_camera_item_acceptor_aspect_id! {}
+}
+impl Aspect for CameraItemAcceptor {
+	impl_aspect_for_camera_item_acceptor_aspect! {}
+}
+impl CameraItemAcceptorAspect for CameraItemAcceptor {
 	fn capture_item(node: Arc<Node>, _calling_client: Arc<Client>, item: Arc<Node>) -> Result<()> {
 		super::acceptor_capture_item_flex(node, item)
 	}
 }
 
-pub fn update(token: &MainThreadToken) {
-	for camera in ITEM_TYPE_INFO_CAMERA.items.get_valid_contents() {
-		let ItemType::Camera(camera) = &camera.specialization else {
-			continue;
-		};
-		camera.update(token);
-	}
-}
-
-create_interface!(ItemInterface);
-impl InterfaceAspect for ItemInterface {
+impl InterfaceAspect for Interface {
 	#[doc = "Create a camera item at a specific location"]
 	fn create_camera_item(
 		_node: Arc<Node>,
@@ -206,19 +160,21 @@ impl InterfaceAspect for ItemInterface {
 	}
 
 	#[doc = "Register this client to manage camera items and create default 3D UI for them."]
-	fn register_camera_item_ui(_node: Arc<Node>, calling_client: Arc<Client>) -> Result<()> {
+	fn register_camera_item_ui(node: Arc<Node>, calling_client: Arc<Client>) -> Result<()> {
+		node.add_aspect(CameraItemUi);
 		register_item_ui_flex(calling_client, &ITEM_TYPE_INFO_CAMERA)
 	}
 
 	#[doc = "Create an item acceptor to allow temporary ownership of a given type of item. Creates a node at `/item/camera/acceptor/<name>`."]
 	fn create_camera_item_acceptor(
-		_node: Arc<Node>,
+		node: Arc<Node>,
 		calling_client: Arc<Client>,
 		id: u64,
 		parent: Arc<Node>,
 		transform: Transform,
 		field: Arc<Node>,
 	) -> Result<()> {
+		node.add_aspect(CameraItemAcceptor);
 		create_item_acceptor_flex(
 			calling_client,
 			id,

src/nodes/items/mod.rs

@@ -4,15 +4,16 @@ pub mod panel;
 use self::camera::CameraItem;
 use self::panel::PanelItemTrait;
 use super::alias::AliasList;
-use super::fields::{Field, FIELD_ALIAS_INFO};
+use super::fields::{FIELD_ALIAS_INFO, Field};
 use super::spatial::Spatial;
-use super::{Alias, Aspect, Node};
+use super::{Alias, Aspect, AspectIdentifier, Node};
 use crate::core::client::Client;
+use crate::core::error::Result;
 use crate::core::registry::Registry;
+use crate::ensure;
 use crate::nodes::alias::AliasInfo;
-use crate::nodes::spatial::Transform;
 use crate::nodes::spatial::SPATIAL_ASPECT_ALIAS_INFO;
-use color_eyre::eyre::{ensure, Result};
+use crate::nodes::spatial::Transform;
 use parking_lot::Mutex;
 use std::hash::Hash;
 use std::sync::{Arc, Weak};
@@ -47,6 +48,8 @@ pub struct TypeInfo {
 	pub ui: Mutex<Weak<ItemUI>>,
 	pub items: Registry<Item>,
 	pub acceptors: Registry<ItemAcceptor>,
+	pub add_ui_aspect: fn(node: &Node),
+	pub add_acceptor_aspect: fn(node: &Node),
 	pub new_acceptor_fn: fn(node: &Node, acceptor: &Arc<Node>, acceptor_field: &Arc<Node>),
 }
 impl Hash for TypeInfo {
@@ -81,7 +84,6 @@ impl Item {
 		};
 		let item = type_info.items.add(item);
 
-		<Item as ItemAspect>::add_node_members(node);
 		if let Some(ui) = type_info.ui.lock().upgrade() {
 			ui.handle_create_item(&item);
 		}
@@ -108,8 +110,11 @@ impl Item {
 		)
 	}
 }
+impl AspectIdentifier for Item {
+	impl_aspect_for_item_aspect_id! {}
+}
 impl Aspect for Item {
-	const NAME: &'static str = "Item";
+	impl_aspect_for_item_aspect! {}
 }
 impl ItemAspect for Item {
 	fn release(node: Arc<Node>, _calling_client: Arc<Client>) -> Result<()> {
@@ -128,8 +133,9 @@ impl Drop for Item {
 	}
 }
 
+#[cfg_attr(not(feature = "wayland"), allow(dead_code))]
 pub enum ItemType {
-	Camera(CameraItem),
+	Camera(Arc<CameraItem>),
 	Panel(Arc<dyn PanelItemTrait>),
 }
 impl ItemType {
@@ -284,8 +290,11 @@ impl ItemUI {
 			.remove_aspect(acceptor.field.as_ref());
 	}
 }
+impl AspectIdentifier for ItemUI {
+	impl_aspect_for_item_ui_aspect_id! {}
+}
 impl Aspect for ItemUI {
-	const NAME: &'static str = "Item";
+	impl_aspect_for_item_ui_aspect! {}
 }
 impl Drop for ItemUI {
 	fn drop(&mut self) {
@@ -342,8 +351,11 @@ impl ItemAcceptor {
 		let _ = item_acceptor_client::release_item(&node, alias.id);
 	}
 }
+impl AspectIdentifier for ItemAcceptor {
+	impl_aspect_for_item_acceptor_aspect_id! {}
+}
 impl Aspect for ItemAcceptor {
-	const NAME: &'static str = "ItemAcceptor";
+	impl_aspect_for_item_acceptor_aspect! {}
 }
 impl ItemAcceptorAspect for ItemAcceptor {}
 impl Drop for ItemAcceptor {
@@ -364,6 +376,7 @@ pub fn register_item_ui_flex(
 ) -> Result<()> {
 	let ui = Node::from_id(&calling_client, type_info.ui_node_id, true).add_to_scenegraph()?;
 	ItemUI::add_to(&ui, type_info)?;
+	(type_info.add_ui_aspect)(&ui);
 	Ok(())
 }
 fn create_item_acceptor_flex(
@@ -381,6 +394,7 @@ fn create_item_acceptor_flex(
 	let node = Node::from_id(&calling_client, id, true).add_to_scenegraph()?;
 	Spatial::add_to(&node, Some(space.clone()), transform, false);
 	ItemAcceptor::add_to(&node, type_info, field);
+	(type_info.add_acceptor_aspect)(&node);
 	Ok(())
 }
 
@@ -391,6 +405,3 @@ fn acceptor_capture_item_flex(node: Arc<Node>, item: Arc<Node>) -> Result<()> {
 
 	Ok(())
 }
-
-struct ItemInterface;
-// create_interface!(ItemInterface);

src/nodes/items/panel.rs

@@ -1,25 +1,29 @@
-use super::{create_item_acceptor_flex, register_item_ui_flex, ItemAcceptor, ItemInterface};
+use super::camera::CameraItemAcceptor;
+use super::{create_item_acceptor_flex, register_item_ui_flex};
+use crate::bail;
+use crate::core::error::Result;
 use crate::nodes::items::ITEM_ACCEPTOR_ASPECT_ALIAS_INFO;
 use crate::nodes::items::ITEM_ASPECT_ALIAS_INFO;
+use crate::nodes::{Aspect, AspectIdentifier};
 use crate::{
 	core::{
-		client::{get_env, state, Client, INTERNAL_CLIENT},
+		client::{Client, INTERNAL_CLIENT, get_env, state},
 		registry::Registry,
 	},
-	create_interface,
 	nodes::{
+		Node,
 		drawable::model::ModelPart,
 		items::{Item, ItemType, TypeInfo},
 		spatial::{Spatial, Transform},
-		Node,
 	},
 };
-use color_eyre::eyre::Result;
 use glam::Mat4;
 use lazy_static::lazy_static;
 use mint::Vector2;
+use parking_lot::Mutex;
+use slotmap::{DefaultKey, Key, KeyData, SlotMap};
 use std::sync::{Arc, Weak};
-use tracing::{debug, info};
+use tracing::debug;
 
 stardust_xr_server_codegen::codegen_item_panel_protocol!();
 impl Default for Geometry {
@@ -30,8 +34,10 @@ impl Default for Geometry {
 		}
 	}
 }
+impl Copy for Geometry {}
 
 lazy_static! {
+	pub static ref KEYMAPS: Mutex<SlotMap<DefaultKey, String>> = Mutex::new(SlotMap::default());
 	pub static ref ITEM_TYPE_INFO_PANEL: TypeInfo = TypeInfo {
 		type_name: "panel",
 		alias_info: PANEL_ITEM_ASPECT_ALIAS_INFO.clone(),
@@ -39,6 +45,12 @@ lazy_static! {
 		ui: Default::default(),
 		items: Registry::new(),
 		acceptors: Registry::new(),
+		add_acceptor_aspect: |node| {
+			node.add_aspect(PanelItemUi);
+		},
+		add_ui_aspect: |node| {
+			node.add_aspect(PanelItemAcceptor);
+		},
 		new_acceptor_fn: |node, acceptor, acceptor_field| {
 			let _ = panel_item_ui_client::create_acceptor(node, acceptor, acceptor_field);
 		}
@@ -65,7 +77,7 @@ pub trait Backend: Send + Sync + 'static {
 		scroll_steps: Option<Vector2<f32>>,
 	);
 
-	fn keyboard_keys(&self, surface: &SurfaceId, keymap_id: u64, keys: Vec<i32>);
+	fn keyboard_key(&self, surface: &SurfaceId, keymap_id: u64, key: u32, pressed: bool);
 
 	fn touch_down(&self, surface: &SurfaceId, id: u32, position: Vector2<f32>);
 	fn touch_move(&self, id: u32, position: Vector2<f32>);
@@ -86,11 +98,13 @@ pub trait PanelItemTrait: Send + Sync + 'static {
 	fn send_acceptor_item_created(&self, node: &Node, item: &Arc<Node>);
 }
 
+#[derive(Debug)]
 pub struct PanelItem<B: Backend> {
 	pub node: Weak<Node>,
 	pub backend: Box<B>,
 }
 impl<B: Backend> PanelItem<B> {
+	#[cfg_attr(not(feature = "wayland"), allow(dead_code))]
 	pub fn create(backend: Box<B>, pid: Option<i32>) -> (Arc<Node>, Arc<PanelItem<B>>) {
 		debug!(?pid, "Create panel item");
 
@@ -115,7 +129,7 @@ impl<B: Backend> PanelItem<B> {
 			&ITEM_TYPE_INFO_PANEL,
 			ItemType::Panel(generic_panel_item),
 		);
-		<Self as PanelItemAspect>::add_node_members(&node);
+		node.add_aspect_raw(panel_item.clone());
 
 		(node, panel_item)
 	}
@@ -197,9 +211,12 @@ impl<B: Backend> PanelItem<B> {
 		panel_item_client::destroy_child(&node, id);
 	}
 }
-
-// make these stupid vectors u32 in the protocol somehow!!!!!!!1
-
+impl<B: Backend> AspectIdentifier for PanelItem<B> {
+	impl_aspect_for_panel_item_aspect_id! {}
+}
+impl<B: Backend> Aspect for PanelItem<B> {
+	impl_aspect_for_panel_item_aspect! {}
+}
 #[allow(unused)]
 impl<B: Backend> PanelItemAspect for PanelItem<B> {
 	#[doc = "Apply the cursor as a material to a model."]
@@ -341,19 +358,20 @@ impl<B: Backend> PanelItemAspect for PanelItem<B> {
 	}
 
 	#[doc = "Send a series of key presses and releases (positive keycode for pressed, negative for released)."]
-	fn keyboard_keys(
+	fn keyboard_key(
 		node: Arc<Node>,
 		_calling_client: Arc<Client>,
 		surface: SurfaceId,
 		keymap_id: u64,
-		keys: Vec<i32>,
+		key: u32,
+		pressed: bool,
 	) -> Result<()> {
 		let Some(panel_item) = panel_item_from_node(&node) else {
 			return Ok(());
 		};
 		panel_item
 			.backend()
-			.keyboard_keys(&surface, keymap_id, keys);
+			.keyboard_key(&surface, keymap_id, key, pressed);
 		Ok(())
 	}
 
@@ -405,7 +423,23 @@ impl<B: Backend> PanelItemAspect for PanelItem<B> {
 	}
 }
 
-impl PanelItemAcceptorAspect for ItemAcceptor {
+pub struct PanelItemUi;
+impl AspectIdentifier for PanelItemUi {
+	impl_aspect_for_panel_item_ui_aspect_id! {}
+}
+impl Aspect for PanelItemUi {
+	impl_aspect_for_panel_item_ui_aspect! {}
+}
+impl PanelItemUiAspect for PanelItemUi {}
+
+pub struct PanelItemAcceptor;
+impl AspectIdentifier for PanelItemAcceptor {
+	impl_aspect_for_panel_item_acceptor_aspect_id! {}
+}
+impl Aspect for PanelItemAcceptor {
+	impl_aspect_for_panel_item_acceptor_aspect! {}
+}
+impl PanelItemAcceptorAspect for PanelItemAcceptor {
 	fn capture_item(node: Arc<Node>, _calling_client: Arc<Client>, item: Arc<Node>) -> Result<()> {
 		super::acceptor_capture_item_flex(node, item)
 	}
@@ -428,29 +462,24 @@ impl<B: Backend> PanelItemTrait for PanelItem<B> {
 		let _ = panel_item_acceptor_client::capture_item(node, item, init_data);
 	}
 }
-impl<B: Backend> Drop for PanelItem<B> {
-	fn drop(&mut self) {
-		// Dropped panel item, basically just a debug breakpoint place
-		info!("Dropped panel item");
-	}
-}
 
-create_interface!(ItemInterface);
-impl InterfaceAspect for ItemInterface {
+impl InterfaceAspect for Interface {
 	#[doc = "Register this client to manage the items of a certain type and create default 3D UI for them."]
-	fn register_panel_item_ui(_node: Arc<Node>, calling_client: Arc<Client>) -> Result<()> {
+	fn register_panel_item_ui(node: Arc<Node>, calling_client: Arc<Client>) -> Result<()> {
+		node.add_aspect(CameraItemAcceptor);
 		register_item_ui_flex(calling_client, &ITEM_TYPE_INFO_PANEL)
 	}
 
 	#[doc = "Create an item acceptor to allow temporary ownership of a given type of item. Creates a node at `/item/<item_type>/acceptor/<name>`."]
 	fn create_panel_item_acceptor(
-		_node: Arc<Node>,
+		node: Arc<Node>,
 		calling_client: Arc<Client>,
 		id: u64,
 		parent: Arc<Node>,
 		transform: Transform,
 		field: Arc<Node>,
 	) -> Result<()> {
+		node.add_aspect(PanelItemAcceptor);
 		create_item_acceptor_flex(
 			calling_client,
 			id,
@@ -460,4 +489,36 @@ impl InterfaceAspect for ItemInterface {
 			field,
 		)
 	}
+
+	async fn register_keymap(
+		_node: Arc<Node>,
+		_calling_client: Arc<Client>,
+		keymap: String,
+	) -> Result<u64> {
+		let mut keymaps = KEYMAPS.lock();
+		if let Some(found_keymap_id) = keymaps
+			.iter()
+			.filter(|(_k, v)| *v == &keymap)
+			.map(|(k, _v)| k)
+			.last()
+		{
+			return Ok(found_keymap_id.data().as_ffi());
+		}
+
+		let key = keymaps.insert(keymap);
+		Ok(key.data().as_ffi())
+	}
+
+	async fn get_keymap(
+		_node: Arc<Node>,
+		_calling_client: Arc<Client>,
+		keymap_id: u64,
+	) -> Result<String> {
+		let keymaps = KEYMAPS.lock();
+		let Some(keymap) = keymaps.get(KeyData::from_ffi(keymap_id).into()) else {
+			bail!("Could not find keymap. Try registering it");
+		};
+
+		Ok(keymap.clone())
+	}
 }

src/nodes/mod.rs

@@ -1,6 +1,5 @@
 pub mod alias;
 pub mod audio;
-pub mod data;
 pub mod drawable;
 pub mod fields;
 pub mod input;
@@ -10,13 +9,11 @@ pub mod spatial;
 
 use self::alias::Alias;
 use crate::core::client::Client;
+use crate::core::error::{Result, ServerError};
 use crate::core::registry::Registry;
 use crate::core::scenegraph::MethodResponseSender;
-use color_eyre::eyre::{eyre, Result};
-use parking_lot::Mutex;
-use portable_atomic::{AtomicBool, Ordering};
-use rustc_hash::FxHashMap;
-use serde::{de::DeserializeOwned, Serialize};
+use dashmap::DashMap;
+use serde::{Serialize, de::DeserializeOwned};
 use spatial::Spatial;
 use stardust_xr::messenger::MessageSenderHandle;
 use stardust_xr::scenegraph::ScenegraphError;
@@ -24,6 +21,7 @@ use stardust_xr::schemas::flex::{deserialize, serialize};
 use std::any::{Any, TypeId};
 use std::fmt::Debug;
 use std::os::fd::OwnedFd;
+use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, Weak};
 use std::vec::Vec;
 
@@ -46,11 +44,29 @@ impl AsRef<[u8]> for Message {
 	}
 }
 
-pub type Signal = fn(Arc<Node>, Arc<Client>, Message) -> Result<()>;
-pub type Method = fn(Arc<Node>, Arc<Client>, Message, MethodResponseSender);
-
 stardust_xr_server_codegen::codegen_node_protocol!();
 
+pub struct Owned;
+impl AspectIdentifier for Owned {
+	impl_aspect_for_owned_aspect_id! {}
+}
+impl Aspect for Owned {
+	impl_aspect_for_owned_aspect! {}
+}
+impl OwnedAspect for Owned {
+	fn set_enabled(node: Arc<Node>, _calling_client: Arc<Client>, enabled: bool) -> Result<()> {
+		node.set_enabled(enabled);
+		Ok(())
+	}
+
+	fn destroy(node: Arc<Node>, _calling_client: Arc<Client>) -> Result<()> {
+		if node.destroyable {
+			node.destroy();
+		}
+		Ok(())
+	}
+}
+
 pub struct OwnedNode(pub Arc<Node>);
 impl Drop for OwnedNode {
 	fn drop(&mut self) {
@@ -64,8 +80,6 @@ pub struct Node {
 	client: Weak<Client>,
 	message_sender_handle: Option<MessageSenderHandle>,
 
-	local_signals: Mutex<FxHashMap<u64, Signal>>,
-	local_methods: Mutex<FxHashMap<u64, Method>>,
 	aliases: Registry<Alias>,
 	aspects: Aspects,
 	destroyable: bool,
@@ -87,26 +101,24 @@ impl Node {
 			client: Arc::downgrade(client),
 			message_sender_handle: client.message_sender_handle.clone(),
 			id,
-			local_signals: Default::default(),
-			local_methods: Default::default(),
 			aliases: Default::default(),
 			aspects: Default::default(),
 			destroyable,
 		};
-		<Node as OwnedAspect>::add_node_members(&node);
+		node.aspects.add(Owned);
 		node
 	}
 	pub fn add_to_scenegraph(self) -> Result<Arc<Node>> {
 		Ok(self
 			.get_client()
-			.ok_or_else(|| eyre!("Internal: Unable to get client"))?
+			.ok_or(ServerError::NoClient)?
 			.scenegraph
 			.add_node(self))
 	}
 	pub fn add_to_scenegraph_owned(self) -> Result<OwnedNode> {
 		Ok(OwnedNode(
 			self.get_client()
-				.ok_or_else(|| eyre!("Internal: Unable to get client"))?
+				.ok_or(ServerError::NoClient)?
 				.scenegraph
 				.add_node(self),
 		))
@@ -118,7 +130,8 @@ impl Node {
 					.global_transform()
 					.to_scale_rotation_translation()
 					.0
-					.length_squared() > 0.0
+					.length_squared()
+					> 0.0
 			} else {
 				true
 			}
@@ -146,46 +159,42 @@ impl Node {
 	// 	Ok(serialize(pid)?.into())
 	// }
 
-	pub fn add_local_signal(&self, id: u64, signal: Signal) {
-		self.local_signals.lock().insert(id, signal);
-	}
-	pub fn add_local_method(&self, id: u64, method: Method) {
-		self.local_methods.lock().insert(id, method);
-	}
-
-	pub fn add_aspect<A: Aspect>(&self, aspect: A) -> Arc<A> {
+	pub fn add_aspect<A: AspectIdentifier>(&self, aspect: A) -> Arc<A> {
 		self.aspects.add(aspect)
 	}
-	pub fn add_aspect_raw<A: Aspect>(&self, aspect: Arc<A>) {
+	pub fn add_aspect_raw<A: AspectIdentifier>(&self, aspect: Arc<A>) {
 		self.aspects.add_raw(aspect)
 	}
-	pub fn get_aspect<A: Aspect>(&self) -> Result<Arc<A>> {
+	pub fn get_aspect<A: AspectIdentifier>(&self) -> Result<Arc<A>> {
 		self.aspects.get()
 	}
 
 	pub fn send_local_signal(
 		self: Arc<Self>,
 		calling_client: Arc<Client>,
+		aspect_id: u64,
 		method: u64,
 		message: Message,
 	) -> Result<(), ScenegraphError> {
 		if let Ok(alias) = self.get_aspect::<Alias>() {
-			if !alias.info.server_signals.iter().any(|e| *e == method) {
-				return Err(ScenegraphError::SignalNotFound);
+			if !alias.info.server_signals.contains(&method) {
+				return Err(ScenegraphError::MemberNotFound);
 			}
 			alias
 				.original
 				.upgrade()
 				.ok_or(ScenegraphError::BrokenAlias)?
-				.send_local_signal(calling_client, method, message)
+				.send_local_signal(calling_client, aspect_id, method, message)
 		} else {
-			let signal = self
-				.local_signals
-				.lock()
-				.get(&method)
-				.cloned()
-				.ok_or(ScenegraphError::SignalNotFound)?;
-			signal(self, calling_client, message).map_err(|error| ScenegraphError::SignalError {
+			let aspect = self
+				.aspects
+				.0
+				.get(&aspect_id)
+				.ok_or(ScenegraphError::AspectNotFound)?
+				.clone();
+			aspect
+				.run_signal(calling_client, self.clone(), method, message)
+				.map_err(|error| ScenegraphError::MemberError {
 					error: error.to_string(),
 				})
 		}
@@ -193,13 +202,14 @@ impl Node {
 	pub fn execute_local_method(
 		self: Arc<Self>,
 		calling_client: Arc<Client>,
+		aspect_id: u64,
 		method: u64,
 		message: Message,
 		response: MethodResponseSender,
 	) {
 		if let Ok(alias) = self.get_aspect::<Alias>() {
-			if !alias.info.server_methods.iter().any(|e| *e == method) {
-				response.send(Err(ScenegraphError::MethodNotFound));
+			if !alias.info.server_methods.contains(&method) {
+				response.send(Err(ScenegraphError::MemberNotFound));
 				return;
 			}
 			let Some(alias) = alias.original.upgrade() else {
@@ -208,6 +218,7 @@ impl Node {
 			};
 			alias.execute_local_method(
 				calling_client,
+				aspect_id,
 				method,
 				Message {
 					data: message.data.clone(),
@@ -216,14 +227,19 @@ impl Node {
 				response,
 			)
 		} else {
-			let Some(method) = self.local_methods.lock().get(&method).cloned() else {
-				response.send(Err(ScenegraphError::MethodNotFound));
+			let Some(aspect) = self.aspects.0.get(&aspect_id).map(|v| v.clone()) else {
+				response.send(Err(ScenegraphError::AspectNotFound));
 				return;
 			};
-			method(self, calling_client, message, response);
+			aspect.run_method(calling_client, self.clone(), method, message, response);
 		}
 	}
-	pub fn send_remote_signal(&self, method: u64, message: impl Into<Message>) -> Result<()> {
+	pub fn send_remote_signal(
+		&self,
+		aspect_id: u64,
+		method: u64,
+		message: impl Into<Message>,
+	) -> Result<()> {
 		let message = message.into();
 		self.aliases
 			.get_valid_contents()
@@ -233,6 +249,7 @@ impl Node {
 			.for_each(|node| {
 				// Beware! file descriptors will not be sent to aliases!!!
 				let _ = node.send_remote_signal(
+					aspect_id,
 					method,
 					Message {
 						data: message.data.clone(),
@@ -241,12 +258,13 @@ impl Node {
 				);
 			});
 		if let Some(handle) = self.message_sender_handle.as_ref() {
-			handle.signal(self.id, method, &message.data, message.fds)?;
+			handle.signal(self.id, aspect_id, method, &message.data, message.fds)?;
 		}
 		Ok(())
 	}
 	pub async fn execute_remote_method_typed<S: Serialize, D: DeserializeOwned>(
 		&self,
+		aspect_id: u64,
 		method: u64,
 		input: S,
 		fds: Vec<OwnedFd>,
@@ -254,13 +272,13 @@ impl Node {
 		let message_sender_handle = self
 			.message_sender_handle
 			.as_ref()
-			.ok_or(eyre!("Messenger does not exist for this node"))?;
+			.ok_or(ServerError::NoMessenger)?;
 
 		let serialized = serialize(input)?;
 		let result = message_sender_handle
-			.method(self.id, method, &serialized, fds)?
-			.await
-			.map_err(|e| eyre!(e))?;
+			.method(self.id, aspect_id, method, &serialized, fds)
+			.await?
+			.map_err(ServerError::RemoteMethodError)?;
 
 		let (message, fds) = result.into_components();
 		let deserialized: D = deserialize(&message)?;
@@ -271,61 +289,62 @@ impl Debug for Node {
 	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
 		f.debug_struct("Node")
 			.field("id", &self.id)
-			.field("local_signals", &self.local_signals.lock().keys())
-			.field("local_methods", &self.local_methods.lock().keys())
 			.field("destroyable", &self.destroyable)
 			.finish()
 	}
 }
-impl OwnedAspect for Node {
-	fn set_enabled(node: Arc<Node>, _calling_client: Arc<Client>, enabled: bool) -> Result<()> {
-		node.set_enabled(enabled);
-		Ok(())
-	}
-
-	fn destroy(node: Arc<Node>, _calling_client: Arc<Client>) -> Result<()> {
-		if node.destroyable {
-			node.destroy();
-		}
-		Ok(())
-	}
-}
 impl Drop for Node {
 	fn drop(&mut self) {
 		// Debug breakpoint
 	}
 }
 
+pub trait AspectIdentifier: Aspect {
+	const ID: u64;
+}
 pub trait Aspect: Any + Send + Sync + 'static {
-	const NAME: &'static str;
+	fn as_any(self: Arc<Self>) -> Arc<dyn Any + Send + Sync + 'static>;
+	fn run_signal(
+		&self,
+		calling_client: Arc<Client>,
+		node: Arc<Node>,
+		signal: u64,
+		message: Message,
+	) -> Result<(), stardust_xr::scenegraph::ScenegraphError>;
+	fn run_method(
+		&self,
+		calling_client: Arc<Client>,
+		node: Arc<Node>,
+		method: u64,
+		message: Message,
+		response: MethodResponseSender,
+	);
 }
 
 #[derive(Default)]
-struct Aspects(Mutex<FxHashMap<TypeId, Arc<dyn Any + Send + Sync + 'static>>>);
-
+struct Aspects(DashMap<u64, Arc<dyn Aspect>>);
 impl Aspects {
-	fn add<A: Aspect>(&self, t: A) -> Arc<A> {
+	fn add<A: AspectIdentifier>(&self, t: A) -> Arc<A> {
 		let aspect = Arc::new(t);
 		self.add_raw(aspect.clone());
 		aspect
 	}
-	fn add_raw<A: Aspect>(&self, aspect: Arc<A>) {
-		self.0.lock().insert(Self::type_key::<A>(), aspect);
+	fn add_raw<A: AspectIdentifier>(&self, aspect: Arc<A>) {
+		self.0.insert(A::ID, aspect);
 	}
-	fn get<A: Aspect>(&self) -> Result<Arc<A>> {
+	fn get<A: Aspect + AspectIdentifier>(&self) -> Result<Arc<A>> {
 		self.0
-			.lock()
-			.get(&Self::type_key::<A>())
-			.and_then(|a| Arc::downcast(a.clone()).ok())
-			.ok_or(eyre!("Couldn't get aspect {}", A::NAME.to_lowercase()))
-	}
-
-	fn type_key<A: 'static>() -> TypeId {
-		TypeId::of::<A>()
+			.get(&A::ID)
+			// .cloned doesn't work for some reason
+			.map(|v| v.clone())
+			.map(|a| a.as_any())
+			.and_then(|a| Arc::downcast(a).ok())
+			.ok_or(ServerError::NoAspect(TypeId::of::<A>()))
 	}
 }
 impl Drop for Aspects {
 	fn drop(&mut self) {
-		self.0.lock().clear()
+		// why would this be needed? do drop impls not run otherwise?
+		self.0.clear()
 	}
 }

src/nodes/root.rs

@@ -1,19 +1,17 @@
 use super::spatial::Spatial;
-use super::Node;
-use crate::core::client::Client;
+use super::{Aspect, AspectIdentifier, Node};
+use crate::bail;
+use crate::core::client::{CLIENTS, Client};
 use crate::core::client_state::ClientStateParsed;
-use crate::core::registry::Registry;
+use crate::core::error::Result;
 use crate::nodes::spatial::SPATIAL_REF_ASPECT_ALIAS_INFO;
 use crate::session::connection_env;
-use color_eyre::eyre::{bail, Result};
 use glam::Mat4;
 use std::path::PathBuf;
 use std::sync::Arc;
 use std::time::Instant;
 use tracing::info;
 
-static ROOT_REGISTRY: Registry<Root> = Registry::new();
-
 stardust_xr_server_codegen::codegen_root_protocol!();
 
 pub struct Root {
@@ -23,18 +21,20 @@ pub struct Root {
 impl Root {
 	pub fn create(client: &Arc<Client>, transform: Mat4) -> Result<Arc<Self>> {
 		let node = Node::from_id(client, 0, false);
-		<Self as RootAspect>::add_node_members(&node);
 		let node = node.add_to_scenegraph()?;
 		let _ = Spatial::add_to(&node, None, transform, false);
-
-		Ok(ROOT_REGISTRY.add(Root {
-			node,
+		let root_aspect = node.add_aspect(Root {
+			node: node.clone(),
 			connect_instant: Instant::now(),
-		}))
+		});
+		Ok(root_aspect)
 	}
 
 	pub fn send_frame_events(delta: f64) {
-		for root in ROOT_REGISTRY.get_valid_contents() {
+		for client in CLIENTS.get_vec() {
+			let Some(root) = client.root.get() else {
+				continue;
+			};
 			let _ = root_client::frame(
 				&root.node,
 				&FrameInfo {
@@ -47,13 +47,19 @@ impl Root {
 
 	pub fn set_transform(&self, transform: Mat4) {
 		let spatial = self.node.get_aspect::<Spatial>().unwrap();
-		spatial.set_spatial_parent(None).unwrap();
+		// spatial.set_spatial_parent(None).unwrap();
 		spatial.set_local_transform(transform);
 	}
 	pub async fn save_state(&self) -> Result<ClientState> {
 		Ok(root_client::save_state(&self.node).await?.0)
 	}
 }
+impl AspectIdentifier for Root {
+	impl_aspect_for_root_aspect_id! {}
+}
+impl Aspect for Root {
+	impl_aspect_for_root_aspect! {}
+}
 impl RootAspect for Root {
 	async fn get_state(_node: Arc<Node>, calling_client: Arc<Client>) -> Result<ClientState> {
 		let Some(state) = calling_client.state.get() else {
@@ -92,13 +98,8 @@ impl RootAspect for Root {
 	}
 
 	#[doc = "Cleanly disconnect from the server"]
-	fn disconnect(_node: Arc<Node>, calling_client: Arc<Client>) -> color_eyre::eyre::Result<()> {
+	fn disconnect(_node: Arc<Node>, calling_client: Arc<Client>) -> Result<()> {
 		calling_client.disconnect(Ok(()));
 		Ok(())
 	}
 }
-impl Drop for Root {
-	fn drop(&mut self) {
-		ROOT_REGISTRY.remove(self);
-	}
-}

src/nodes/spatial/mod.rs

@@ -3,21 +3,91 @@ pub mod zone;
 use self::zone::Zone;
 use super::alias::Alias;
 use super::fields::{Field, FieldTrait};
-use super::Aspect;
+use super::{Aspect, AspectIdentifier};
+use crate::bail;
 use crate::core::client::Client;
+use crate::core::error::Result;
 use crate::core::registry::Registry;
-use crate::create_interface;
 use crate::nodes::{Node, OWNED_ASPECT_ALIAS_INFO};
-use color_eyre::eyre::{eyre, OptionExt, Result};
-use glam::{vec3a, Mat4, Quat, Vec3};
+use bevy::prelude::Transform as BevyTransform;
+use bevy::prelude::*;
+use bevy::render::primitives::Aabb;
+use color_eyre::eyre::OptionExt;
+use glam::{Mat4, Quat, Vec3, vec3a};
 use mint::Vector3;
-use once_cell::sync::OnceCell;
 use parking_lot::Mutex;
 use rustc_hash::FxHashMap;
 use std::fmt::Debug;
-use std::ptr;
-use std::sync::{Arc, Weak};
-use stereokit_rust::maths::Bounds;
+use std::sync::atomic::Ordering;
+use std::sync::{Arc, OnceLock, Weak};
+use std::{f32, ptr};
+
+pub struct SpatialNodePlugin;
+impl Plugin for SpatialNodePlugin {
+	fn build(&self, app: &mut App) {
+		app.add_systems(
+			PostUpdate,
+			update_spatial_nodes.before(TransformSystem::TransformPropagate),
+		);
+	}
+}
+
+fn update_spatial_nodes(
+	mut query: Query<(
+		&mut BevyTransform,
+		&SpatialNode,
+		Option<&ChildOf>,
+		&mut Visibility,
+	)>,
+	parent_query: Query<&GlobalTransform>,
+) {
+	query
+		.par_iter_mut()
+		.for_each(|(mut transform, spatial_node, child_of, mut vis)| {
+			let _span = debug_span!("updating spatial node").entered();
+			let Some(spatial) = spatial_node.0.upgrade() else {
+				return;
+			};
+			if spatial
+				.node()
+				.is_some_and(|v| !v.enabled.load(Ordering::Relaxed))
+			{
+				if !matches!(*vis, Visibility::Hidden) {
+					*vis = Visibility::Hidden;
+				}
+				return;
+			}
+			let mat4 =
+				debug_span!("getting global transform").in_scope(|| spatial.global_transform());
+			let (scale, _, _) = mat4.to_scale_rotation_translation();
+			match (*vis, scale == Vec3::ZERO) {
+				(Visibility::Inherited | Visibility::Visible, true) => {
+					*vis = Visibility::Hidden;
+				}
+				(Visibility::Hidden, false) => {
+					*vis = Visibility::Inherited;
+				}
+				_ => {}
+			}
+			match child_of {
+				Some(child_of) => {
+					let Ok(parent) = parent_query.get(child_of.0) else {
+						warn!("SpatialNode bevy Parent doesn't have global transform");
+						return;
+					};
+					*transform =
+						BevyTransform::from_matrix(parent.compute_matrix().inverse() * mat4);
+				}
+				None => {
+					*transform = BevyTransform::from_matrix(mat4);
+				}
+			}
+		});
+}
+
+#[derive(Clone, Component, Debug)]
+#[require(BevyTransform, Visibility)]
+pub struct SpatialNode(pub Weak<Spatial>);
 
 stardust_xr_server_codegen::codegen_spatial_protocol!();
 impl Transform {
@@ -38,9 +108,15 @@ impl Transform {
 		Mat4::from_scale_rotation_translation(scale.into(), rotation.into(), position.into())
 	}
 }
+impl AspectIdentifier for Zone {
+	impl_aspect_for_zone_aspect_id! {}
+}
+impl Aspect for Zone {
+	impl_aspect_for_zone_aspect! {}
+}
 
 lazy_static::lazy_static! {
-	pub static ref EXPORTED_SPATIALS: Mutex<FxHashMap<u64, Arc<Node>>> = Mutex::new(FxHashMap::default());
+	pub static ref EXPORTED_SPATIALS: Mutex<FxHashMap<u64, Weak<Node>>> = Mutex::new(FxHashMap::default());
 }
 
 static ZONEABLE_REGISTRY: Registry<Spatial> = Registry::new();
@@ -52,7 +128,7 @@ pub struct Spatial {
 	transform: Mutex<Mat4>,
 	zone: Mutex<Weak<Zone>>,
 	children: Registry<Spatial>,
-	pub bounding_box_calc: OnceCell<fn(&Node) -> Bounds>,
+	pub bounding_box_calc: OnceLock<fn(&Node) -> Aabb>,
 }
 
 impl Spatial {
@@ -64,7 +140,7 @@ impl Spatial {
 			transform: Mutex::new(transform),
 			zone: Mutex::new(Weak::new()),
 			children: Registry::new(),
-			bounding_box_calc: OnceCell::default(),
+			bounding_box_calc: OnceLock::default(),
 		})
 	}
 	pub fn add_to(
@@ -74,16 +150,14 @@ impl Spatial {
 		zoneable: bool,
 	) -> Arc<Spatial> {
 		let spatial = Spatial::new(Arc::downgrade(node), parent.clone(), transform);
-		<Spatial as SpatialAspect>::add_node_members(node);
 		if zoneable {
 			ZONEABLE_REGISTRY.add_raw(&spatial);
 		}
 		if let Some(parent) = parent {
 			parent.children.add_raw(&spatial);
 		}
-		<Spatial as SpatialRefAspect>::add_node_members(node);
-		<Spatial as SpatialAspect>::add_node_members(node);
 		node.add_aspect_raw(spatial.clone());
+		node.add_aspect(SpatialRef);
 		spatial
 	}
 
@@ -98,9 +172,9 @@ impl Spatial {
 	}
 
 	// the output bounds are probably way bigger than they need to be
-	pub fn get_bounding_box(&self) -> Bounds {
+	pub fn get_bounding_box(&self) -> Aabb {
 		let Some(node) = self.node() else {
-			return Bounds::default();
+			return Aabb::default();
 		};
 		let mut bounds = self
 			.bounding_box_calc
@@ -108,7 +182,15 @@ impl Spatial {
 			.map(|b| (b)(&node))
 			.unwrap_or_default();
 		for child in self.children.get_valid_contents() {
-			bounds.grown_box(child.get_bounding_box(), child.local_transform());
+			let mat = child.local_transform();
+			let child_aabb = child.get_bounding_box();
+			bounds = Aabb::enclosing([
+				bounds.min().into(),
+				bounds.max().into(),
+				mat.transform_point3(child_aabb.min().into()),
+				mat.transform_point3(child_aabb.max().into()),
+			])
+			.unwrap();
 		}
 		bounds
 	}
@@ -188,45 +270,29 @@ impl Spatial {
 	fn get_parent(&self) -> Option<Arc<Spatial>> {
 		self.parent.lock().clone()
 	}
-	fn set_parent(self: &Arc<Self>, new_parent: Option<&Arc<Spatial>>) {
+	fn set_parent(self: &Arc<Self>, new_parent: &Arc<Spatial>) {
 		if let Some(parent) = self.get_parent() {
 			parent.children.remove(self);
 		}
-		if let Some(new_parent) = &new_parent {
 		new_parent.children.add_raw(self);
+
+		*self.parent.lock() = Some(new_parent.clone());
 	}
 
-		*self.parent.lock() = new_parent.cloned();
-	}
-
-	pub fn set_spatial_parent(self: &Arc<Self>, parent: Option<&Arc<Spatial>>) -> Result<()> {
-		let is_ancestor = parent
-			.as_ref()
-			.map(|parent| self.is_ancestor_of((*parent).clone()))
-			.unwrap_or(false);
-		if is_ancestor {
-			return Err(eyre!("Setting spatial parent would cause a loop"));
+	pub fn set_spatial_parent(self: &Arc<Self>, parent: &Arc<Spatial>) -> Result<()> {
+		if self.is_ancestor_of(parent.clone()) {
+			bail!("Setting spatial parent would cause a loop");
 		}
 		self.set_parent(parent);
 
 		Ok(())
 	}
-	pub fn set_spatial_parent_in_place(
-		self: &Arc<Self>,
-		parent: Option<&Arc<Spatial>>,
-	) -> Result<()> {
-		let is_ancestor = parent
-			.as_ref()
-			.map(|parent| self.is_ancestor_of((*parent).clone()))
-			.unwrap_or(false);
-		if is_ancestor {
-			return Err(eyre!("Setting spatial parent would cause a loop"));
+	pub fn set_spatial_parent_in_place(self: &Arc<Self>, parent: &Arc<Spatial>) -> Result<()> {
+		if self.is_ancestor_of(parent.clone()) {
+			bail!("Setting spatial parent would cause a loop");
 		}
 
-		self.set_local_transform(Spatial::space_to_space_matrix(
-			Some(self),
-			parent.map(AsRef::as_ref),
-		));
+		self.set_local_transform(Spatial::space_to_space_matrix(Some(self), Some(parent)));
 		self.set_parent(parent);
 
 		Ok(())
@@ -238,70 +304,14 @@ impl Spatial {
 			.upgrade()
 			.map(|zone| zone.field.clone())
 			.map(|field| field.distance(self, vec3a(0.0, 0.0, 0.0)))
-			.unwrap_or(f32::MAX)
+			.unwrap_or(f32::NEG_INFINITY)
 	}
 }
+impl AspectIdentifier for Spatial {
+	impl_aspect_for_spatial_aspect_id! {}
+}
 impl Aspect for Spatial {
-	const NAME: &'static str = "Spatial";
-}
-impl SpatialRefAspect for Spatial {
-	async fn get_local_bounding_box(
-		node: Arc<Node>,
-		_calling_client: Arc<Client>,
-	) -> Result<BoundingBox> {
-		let this_spatial = node.get_aspect::<Spatial>()?;
-		let bounds = this_spatial.get_bounding_box();
-
-		Ok(BoundingBox {
-			center: Vec3::from(bounds.center).into(),
-			size: Vec3::from(bounds.dimensions).into(),
-		})
-	}
-
-	async fn get_relative_bounding_box(
-		node: Arc<Node>,
-		_calling_client: Arc<Client>,
-		relative_to: Arc<Node>,
-	) -> Result<BoundingBox> {
-		let this_spatial = node.get_aspect::<Spatial>()?;
-		let relative_spatial = relative_to.get_aspect::<Spatial>()?;
-		let center = Spatial::space_to_space_matrix(Some(&this_spatial), Some(&relative_spatial))
-			.transform_point3([0.0; 3].into());
-		let mut bounds = Bounds {
-			center: center.into(),
-			dimensions: [0.0; 3].into(),
-		};
-		bounds.grown_box(
-			this_spatial.get_bounding_box(),
-			Spatial::space_to_space_matrix(Some(&this_spatial), Some(&relative_spatial)),
-		);
-
-		Ok(BoundingBox {
-			center: Vec3::from(bounds.center).into(),
-			size: Vec3::from(bounds.dimensions).into(),
-		})
-	}
-
-	async fn get_transform(
-		node: Arc<Node>,
-		_calling_client: Arc<Client>,
-		relative_to: Arc<Node>,
-	) -> Result<Transform> {
-		let this_spatial = node.get_aspect::<Spatial>()?;
-		let relative_spatial = relative_to.get_aspect::<Spatial>()?;
-
-		let (scale, rotation, position) = Spatial::space_to_space_matrix(
-			Some(this_spatial.as_ref()),
-			Some(relative_spatial.as_ref()),
-		)
-		.to_scale_rotation_translation();
-
-		Ok(Transform {
-			translation: Some(position.into()),
-			rotation: Some(rotation.into()),
-			scale: Some(scale.into()),
-		})
-	}
+	impl_aspect_for_spatial_aspect! {}
 }
 impl SpatialAspect for Spatial {
 	fn set_local_transform(
@@ -334,7 +344,7 @@ impl SpatialAspect for Spatial {
 		let this_spatial = node.get_aspect::<Spatial>()?;
 		let parent = parent.get_aspect::<Spatial>()?;
 
-		this_spatial.set_spatial_parent(Some(&parent))?;
+		this_spatial.set_spatial_parent(&parent)?;
 		Ok(())
 	}
 
@@ -346,7 +356,7 @@ impl SpatialAspect for Spatial {
 		let this_spatial = node.get_aspect::<Spatial>()?;
 		let parent = parent.get_aspect::<Spatial>()?;
 
-		this_spatial.set_spatial_parent_in_place(Some(&parent))?;
+		this_spatial.set_spatial_parent_in_place(&parent)?;
 		Ok(())
 	}
 
@@ -364,7 +374,7 @@ impl SpatialAspect for Spatial {
 	// legit gotta find a way to remove old ones, this just keeps the node alive
 	async fn export_spatial(node: Arc<Node>, _calling_client: Arc<Client>) -> Result<u64> {
 		let id = rand::random();
-		EXPORTED_SPATIALS.lock().insert(id, node);
+		EXPORTED_SPATIALS.lock().insert(id, Arc::downgrade(&node));
 		Ok(id)
 	}
 }
@@ -389,6 +399,70 @@ impl Drop for Spatial {
 	}
 }
 
+pub struct SpatialRef;
+impl AspectIdentifier for SpatialRef {
+	impl_aspect_for_spatial_ref_aspect_id! {}
+}
+impl Aspect for SpatialRef {
+	impl_aspect_for_spatial_ref_aspect! {}
+}
+impl SpatialRefAspect for SpatialRef {
+	async fn get_local_bounding_box(
+		node: Arc<Node>,
+		_calling_client: Arc<Client>,
+	) -> Result<BoundingBox> {
+		let this_spatial = node.get_aspect::<Spatial>()?;
+		let bounds = this_spatial.get_bounding_box();
+
+		Ok(BoundingBox {
+			center: Vec3::from(bounds.center).into(),
+			size: Vec3::from(bounds.half_extents * 2.0).into(),
+		})
+	}
+
+	async fn get_relative_bounding_box(
+		node: Arc<Node>,
+		_calling_client: Arc<Client>,
+		relative_to: Arc<Node>,
+	) -> Result<BoundingBox> {
+		let this_spatial = node.get_aspect::<Spatial>()?;
+		let relative_spatial = relative_to.get_aspect::<Spatial>()?;
+		let mat = Spatial::space_to_space_matrix(Some(&this_spatial), Some(&relative_spatial));
+		let bb = this_spatial.get_bounding_box();
+		let bounds = Aabb::enclosing([
+			mat.transform_point3(bb.min().into()),
+			mat.transform_point3(bb.max().into()),
+		])
+		.unwrap();
+
+		Ok(BoundingBox {
+			center: Vec3::from(bounds.center).into(),
+			size: Vec3::from(bounds.half_extents * 2.0).into(),
+		})
+	}
+
+	async fn get_transform(
+		node: Arc<Node>,
+		_calling_client: Arc<Client>,
+		relative_to: Arc<Node>,
+	) -> Result<Transform> {
+		let this_spatial = node.get_aspect::<Spatial>()?;
+		let relative_spatial = relative_to.get_aspect::<Spatial>()?;
+
+		let (scale, rotation, position) = Spatial::space_to_space_matrix(
+			Some(this_spatial.as_ref()),
+			Some(relative_spatial.as_ref()),
+		)
+		.to_scale_rotation_translation();
+
+		Ok(Transform {
+			translation: Some(position.into()),
+			rotation: Some(rotation.into()),
+			scale: Some(scale.into()),
+		})
+	}
+}
+
 pub fn parse_transform(transform: Transform, position: bool, rotation: bool, scale: bool) -> Mat4 {
 	let position = position
 		.then_some(transform.translation)
@@ -406,8 +480,7 @@ pub fn parse_transform(transform: Transform, position: bool, rotation: bool, sca
 	Mat4::from_scale_rotation_translation(scale.into(), rotation.into(), position.into())
 }
 
-pub struct SpatialInterface;
-impl InterfaceAspect for SpatialInterface {
+impl InterfaceAspect for Interface {
 	fn create_spatial(
 		_node: Arc<Node>,
 		calling_client: Arc<Client>,
@@ -445,20 +518,19 @@ impl InterfaceAspect for SpatialInterface {
 		calling_client: Arc<Client>,
 		uid: u64,
 	) -> Result<Arc<Node>> {
-		EXPORTED_SPATIALS
+		Ok(EXPORTED_SPATIALS
 			.lock()
 			.get(&uid)
+			.and_then(|s| s.upgrade())
 			.map(|s| {
 				Alias::create(
-					s,
+					&s,
 					&calling_client,
 					SPATIAL_REF_ASPECT_ALIAS_INFO.clone(),
 					None,
 				)
 				.unwrap()
 			})
-			.ok_or_eyre("Couldn't find spatial with that ID")
+			.ok_or_eyre("Couldn't find spatial with that ID")?)
 	}
 }
-
-create_interface!(SpatialInterface);

src/nodes/spatial/zone.rs

@@ -1,23 +1,25 @@
 use super::{
-	Spatial, ZoneAspect, SPATIAL_ASPECT_ALIAS_INFO, SPATIAL_REF_ASPECT_ALIAS_INFO,
-	ZONEABLE_REGISTRY,
+	SPATIAL_ASPECT_ALIAS_INFO, SPATIAL_REF_ASPECT_ALIAS_INFO, Spatial, ZONEABLE_REGISTRY,
+	ZoneAspect,
 };
 use crate::{
-	core::{client::Client, registry::Registry},
+	core::{client::Client, error::Result, registry::Registry},
 	nodes::{
-		alias::{get_original, Alias, AliasList},
+		Node,
+		alias::{Alias, AliasList, get_original},
 		fields::{Field, FieldTrait},
-		Aspect, Node,
 	},
 };
-use color_eyre::eyre::Result;
 use glam::vec3a;
 use std::sync::{Arc, Weak};
 
 pub fn capture(spatial: &Arc<Spatial>, zone: &Arc<Zone>) {
 	let old_distance = spatial.zone_distance();
 	let new_distance = zone.field.distance(spatial, vec3a(0.0, 0.0, 0.0));
-	if new_distance.abs() < old_distance.abs() {
+	if new_distance.abs() > old_distance.abs() {
+		return;
+	}
+
 	release(spatial);
 	*spatial.old_parent.lock() = spatial.get_parent();
 	*spatial.zone.lock() = Arc::downgrade(zone);
@@ -37,14 +39,16 @@ pub fn capture(spatial: &Arc<Spatial>, zone: &Arc<Zone>) {
 	};
 	let _ = super::zone_client::capture(&zone_node, &spatial_alias);
 }
-}
 pub fn release(spatial: &Spatial) {
 	let Some(spatial_node) = spatial.node.upgrade() else {
 		return;
 	};
 	let spatial = spatial_node.get_aspect::<Spatial>().unwrap();
 
-	let _ = spatial.set_spatial_parent_in_place(spatial.old_parent.lock().take().as_ref());
+	let Some(old_parent) = spatial.old_parent.lock().take() else {
+		return;
+	};
+	let _ = spatial.set_spatial_parent_in_place(&old_parent);
 	let mut spatial_zone = spatial.zone.lock();
 
 	if let Some(spatial_zone) = spatial_zone.upgrade() {
@@ -73,7 +77,6 @@ impl Zone {
 			intersecting: AliasList::default(),
 			captured: AliasList::default(),
 		});
-		<Zone as ZoneAspect>::add_node_members(node);
 		node.add_aspect_raw(zone.clone());
 		zone
 	}
@@ -82,16 +85,18 @@ impl Zone {
 
 		let current_zoneables = Registry::new();
 		for zoneable in ZONEABLE_REGISTRY.get_valid_contents() {
+			// Skip if the zoneable is an ancestor of the zone or the zone itself
+			if zoneable.is_ancestor_of(self.spatial.clone()) {
+				continue;
+			}
+
 			let distance = self.field.distance(&zoneable, [0.0; 3].into());
 			if distance > 0.0 {
 				continue;
 			}
-			if let Some(zone) = zoneable.zone.lock().upgrade() {
-				let zoneable_distance = zone.field.distance(&zoneable, [0.0; 3].into());
-				if zoneable_distance < distance {
+			if distance < zoneable.zone_distance() {
 				continue;
 			}
-			}
 			current_zoneables.add_raw(&zoneable);
 		}
 
@@ -124,9 +129,6 @@ impl Zone {
 		Ok(())
 	}
 }
-impl Aspect for Zone {
-	const NAME: &'static str = "Zone";
-}
 impl ZoneAspect for Zone {
 	fn update(node: Arc<Node>, _calling_client: Arc<Client>) -> Result<()> {
 		let zone = node.get_aspect::<Zone>()?;

src/objects/hmd.rs

@@ -0,0 +1,108 @@
+use std::sync::Arc;
+
+use bevy::prelude::*;
+use bevy_mod_openxr::{
+	helper_traits::{ToQuat as _, ToVec3 as _},
+	resources::OxrFrameState,
+	session::OxrSession,
+};
+use bevy_mod_xr::{
+	session::{XrPreDestroySession, XrSessionCreated, session_running},
+	spaces::{XrPrimaryReferenceSpace, XrSpace},
+};
+use openxr::SpaceLocationFlags;
+
+use crate::{DbusConnection, PreFrameWait, nodes::spatial::Spatial};
+
+use super::{ObjectHandle, SpatialRef, input::mouse_pointer::FlatscreenCam};
+
+pub struct HmdPlugin;
+impl Plugin for HmdPlugin {
+	fn build(&self, app: &mut App) {
+		app.add_systems(XrPreDestroySession, destroy_view_space);
+		app.add_systems(XrSessionCreated, create_view_space);
+		app.add_systems(PreFrameWait, update_xr.run_if(session_running));
+		app.add_systems(PreFrameWait, update_flat.run_if(not(session_running)));
+		app.add_systems(Startup, setup);
+	}
+}
+
+fn setup(connection: Res<DbusConnection>, mut cmds: Commands) {
+	let (spatial, _spatial_handle) = SpatialRef::create(&connection, "/org/stardustxr/HMD");
+	let hmd = Hmd {
+		spatial,
+		_spatial_handle,
+		space: None,
+	};
+	cmds.insert_resource(hmd);
+}
+
+fn create_view_space(session: Res<OxrSession>, mut hmd: ResMut<Hmd>) {
+	let space = session
+		.create_reference_space(openxr::ReferenceSpaceType::VIEW, Transform::IDENTITY)
+		.inspect_err(|err| error!("failed to create View XrSpace"))
+		.ok();
+	hmd.space = space.map(|v| v.0);
+}
+fn destroy_view_space(session: Res<OxrSession>, mut cmds: Commands, mut hmd: ResMut<Hmd>) {
+	let Some(space) = hmd.space.take() else {
+		return;
+	};
+	session.destroy_space(space);
+}
+
+#[derive(Resource)]
+struct Hmd {
+	spatial: Arc<Spatial>,
+	_spatial_handle: ObjectHandle<SpatialRef>,
+	space: Option<XrSpace>,
+}
+
+fn update_flat(cam: Single<&GlobalTransform, With<FlatscreenCam>>, hmd: Res<Hmd>) {
+	// this shouldn't be parented to anything, so global and local spaces should be the same
+	hmd.spatial.set_local_transform(cam.compute_matrix());
+}
+
+fn update_xr(
+	session: Option<Res<OxrSession>>,
+	ref_space: Option<Res<XrPrimaryReferenceSpace>>,
+	hmd: Res<Hmd>,
+	state: Option<Res<OxrFrameState>>,
+) {
+	let (Some(session), Some(view), Some(ref_space), Some(state)) =
+		(session, hmd.space, ref_space, state)
+	else {
+		// tokio::task::spawn({
+		// 	let handle = hmd.tracked_handle.clone();
+		// 	async move {
+		// 		handle.set_tracked(false);
+		// 	}
+		// });
+		return;
+	};
+	// this won't be correct with pipelined rendering
+	let location = session
+		.locate_space(&view, &ref_space, state.predicted_display_time)
+		.inspect_err(|err| error!("Error while Locating OpenXR Stage Space {err}"));
+	if let Ok(location) = location {
+		let is_tracked = location
+			.location_flags
+			.contains(SpaceLocationFlags::POSITION_VALID | SpaceLocationFlags::POSITION_TRACKED)
+			|| location.location_flags.contains(
+				SpaceLocationFlags::ORIENTATION_VALID | SpaceLocationFlags::ORIENTATION_TRACKED,
+			);
+		// tokio::task::spawn({
+		// 	let handle = play_space.tracked_handle.clone();
+		// 	async move {
+		// 		handle.set_tracked(is_tracked);
+		// 	}
+		// });
+		if is_tracked {
+			hmd.spatial
+				.set_local_transform(Mat4::from_rotation_translation(
+					location.pose.orientation.to_quat(),
+					location.pose.position.to_vec3(),
+				));
+		}
+	}
+}

src/objects/input/eye_pointer.rs

@@ -1,18 +1,17 @@
 use crate::{
 	core::client::INTERNAL_CLIENT,
 	nodes::{
-		fields::{FieldTrait, Ray},
-		input::{InputDataType, InputMethod, Pointer, INPUT_HANDLER_REGISTRY},
-		spatial::Spatial,
 		Node, OwnedNode,
+		fields::{FieldTrait, Ray},
+		input::{INPUT_HANDLER_REGISTRY, InputDataType, InputMethod, Pointer},
+		spatial::Spatial,
 	},
 };
 use color_eyre::eyre::Result;
-use glam::{vec3, Mat4};
+use glam::{Mat4, vec3};
 use serde::{Deserialize, Serialize};
 use stardust_xr::values::Datamap;
 use std::sync::Arc;
-use stereokit_rust::system::Input;
 
 #[derive(Default, Deserialize, Serialize)]
 pub struct EyeDatamap {
@@ -49,58 +48,4 @@ impl EyePointer {
 			pointer,
 		})
 	}
-	pub fn update(&self) {
-		let ray = Input::get_eyes();
-		self.spatial
-			.set_local_transform(Mat4::from_rotation_translation(
-				ray.orientation.into(),
-				ray.position.into(),
-			));
-		{
-			// Set pointer input datamap
-			*self.pointer.datamap.lock() = Datamap::from_typed(EyeDatamap { eye: 2 }).unwrap();
-		}
-
-		// send input to all the input handlers that are the closest to the ray as possible
-		let rx = INPUT_HANDLER_REGISTRY
-			.get_valid_contents()
-			.into_iter()
-			// filter out all the disabled handlers
-			.filter(|handler| {
-				let Some(node) = handler.spatial.node() else {
-					return false;
-				};
-				node.enabled()
-			})
-			// ray march to all the enabled handlers' fields
-			.map(|handler| {
-				let result = handler.field.ray_march(Ray {
-					origin: vec3(0.0, 0.0, 0.0),
-					direction: vec3(0.0, 0.0, -1.0),
-					space: self.spatial.clone(),
-				});
-				(vec![handler], result)
-			})
-			// make sure the field isn't at the pointer origin and that it's being hit
-			.filter(|(_, result)| result.deepest_point_distance > 0.01 && result.min_distance < 0.0)
-			// .inspect(|(_, result)| {
-			// 	dbg!(result);
-			// })
-			// now collect all handlers that are same distance if they're the closest
-			.reduce(|(mut handlers_a, result_a), (handlers_b, result_b)| {
-				if (result_a.deepest_point_distance - result_b.deepest_point_distance).abs() < 0.001
-				{
-					// distance is basically the same
-					handlers_a.extend(handlers_b);
-					(handlers_a, result_a)
-				} else if result_a.deepest_point_distance < result_b.deepest_point_distance {
-					(handlers_a, result_a)
-				} else {
-					(handlers_b, result_b)
-				}
-			})
-			.map(|(rx, _)| rx)
-			.unwrap_or_default();
-		self.pointer.set_handler_order(rx.iter());
-	}
 }

src/objects/input/mod.rs

@@ -1,44 +1,47 @@
 pub mod eye_pointer;
 pub mod mouse_pointer;
-pub mod sk_controller;
-pub mod sk_hand;
+pub mod oxr_controller;
+pub mod oxr_hand;
 
 use crate::nodes::{
 	fields::{Field, FieldTrait, Ray},
-	input::{InputDataTrait, InputDataType, InputHandler, InputMethod, INPUT_HANDLER_REGISTRY},
+	input::{INPUT_HANDLER_REGISTRY, InputDataTrait, InputDataType, InputHandler, InputMethod},
 	spatial::Spatial,
 };
 use glam::vec3;
-use std::sync::Arc;
+use std::{
+	collections::VecDeque,
+	sync::{Arc, Weak},
+};
 
 #[derive(Default)]
 pub struct CaptureManager {
-	pub capture: Option<Arc<InputHandler>>,
+	pub capture: Weak<InputHandler>,
 }
 impl CaptureManager {
-	pub fn update_capture(&mut self, pointer: &InputMethod) {
-		if let Some(capture) = &self.capture {
-			if !pointer
-				.internal_capture_requests
+	pub fn update_capture(&mut self, method: &InputMethod) {
+		if let Some(capture) = &self.capture.upgrade() {
+			if !method
+				.capture_attempts
 				.get_valid_contents()
 				.contains(capture)
 			{
-				self.capture.take();
+				self.capture = Weak::new();
 			}
 		}
 	}
 	pub fn set_new_capture(
 		&mut self,
-		pointer: &InputMethod,
+		method: &InputMethod,
 		distance_calculator: DistanceCalculator,
 	) {
-		if self.capture.is_none() {
-			self.capture = find_closest_capture(pointer, distance_calculator);
+		if self.capture.upgrade().is_none() {
+			self.capture = find_closest_capture(method, distance_calculator);
 		}
 	}
 	pub fn apply_capture(&self, method: &InputMethod) {
 		method.captures.clear();
-		if let Some(capture) = &self.capture {
+		if let Some(capture) = &self.capture.upgrade() {
 			method.set_handler_order([capture].into_iter());
 			method.captures.add_raw(capture);
 		}
@@ -50,9 +53,9 @@ type DistanceCalculator = fn(&Arc<Spatial>, &InputDataType, &Field) -> Option<f3
 pub fn find_closest_capture(
 	method: &InputMethod,
 	distance_calculator: DistanceCalculator,
-) -> Option<Arc<InputHandler>> {
+) -> Weak<InputHandler> {
 	method
-		.internal_capture_requests
+		.capture_attempts
 		.get_valid_contents()
 		.into_iter()
 		.filter_map(|h| {
@@ -60,39 +63,31 @@ pub fn find_closest_capture(
 				.map(|dist| (h.clone(), dist))
 		})
 		.min_by(|(_, dist_a), (_, dist_b)| dist_a.partial_cmp(dist_b).unwrap())
-		.map(|(handler, _)| handler)
+		.map(|(handler, _)| Arc::downgrade(&handler))
+		.unwrap_or_default()
 }
 
+/// sorts them greatest to least distance (so you can pop off the closest ones easily)
 pub fn get_sorted_handlers(
 	method: &InputMethod,
 	distance_calculator: DistanceCalculator,
-) -> Vec<Arc<InputHandler>> {
-	INPUT_HANDLER_REGISTRY
+) -> Vec<(Arc<InputHandler>, f32)> {
+	let mut handlers = INPUT_HANDLER_REGISTRY
 		.get_valid_contents()
 		.into_iter()
-		.filter(|handler| handler.spatial.node().map_or(false, |node| node.enabled()))
+		.filter(|handler| handler.spatial.node().is_some_and(|node| node.enabled()))
 		.filter(|handler| {
 			handler
 				.field
 				.spatial
 				.node()
-				.map_or(false, |node| node.enabled())
+				.is_some_and(|node| node.enabled())
 		})
 		.filter_map(|handler| {
 			distance_calculator(&method.spatial, &method.data.lock(), &handler.field)
-				.map(|distance| (vec![handler], distance))
+				.map(|distance| (handler, distance))
 		})
-		.filter(|(_, distance)| *distance > 0.0)
-		.reduce(|(mut handlers_a, distance_a), (handlers_b, distance_b)| {
-			if (distance_a - distance_b).abs() < 0.001 {
-				handlers_a.extend(handlers_b);
-				(handlers_a, distance_a)
-			} else if distance_a < distance_b {
-				(handlers_a, distance_a)
-			} else {
-				(handlers_b, distance_b)
-			}
-		})
-		.map(|(handlers, _)| handlers)
-		.unwrap_or_default()
+		.collect::<Vec<_>>();
+	handlers.sort_by(|(_, dist_a), (_, dist_b)| dist_a.partial_cmp(dist_b).unwrap());
+	handlers
 }

src/objects/input/mouse_pointer.rs

@@ -1,26 +1,129 @@
+use super::{CaptureManager, DistanceCalculator, get_sorted_handlers};
 use crate::{
+	DbusConnection, ObjectRegistryRes,
 	core::client::INTERNAL_CLIENT,
 	nodes::{
-		data::{
-			mask_matches, pulse_receiver_client, PulseSender, KEYMAPS, PULSE_RECEIVER_REGISTRY,
-		},
-		fields::{FieldTrait, Ray},
-		input::{InputDataType, InputHandler, InputMethod, Pointer, INPUT_HANDLER_REGISTRY},
-		spatial::Spatial,
 		Node, OwnedNode,
+		fields::{EXPORTED_FIELDS, Field, FieldTrait, Ray},
+		input::{InputDataType, InputMethod, Pointer},
+		items::panel::KEYMAPS,
+		spatial::Spatial,
 	},
 };
+use bevy::{
+	input::{
+		ButtonState,
+		keyboard::{KeyboardInput, NativeKey, NativeKeyCode},
+		mouse::{MouseMotion, MouseWheel},
+	},
+	prelude::*,
+	window::PrimaryWindow,
+};
 use color_eyre::eyre::Result;
-use glam::{vec3, Mat4, Vec3};
+use glam::{Mat4, Vec3, vec3};
 use mint::Vector2;
 use serde::{Deserialize, Serialize};
 use slotmap::{DefaultKey, Key as SlotKey};
-use stardust_xr::values::Datamap;
+use stardust_xr::{
+	schemas::dbus::{interfaces::FieldRefProxy, object_registry::ObjectRegistry},
+	values::Datamap,
+};
 use std::sync::Arc;
-use stereokit_rust::system::{Input, Key};
-use xkbcommon_rs::{xkb_keymap::CompileFlags, Context, Keymap, KeymapFormat};
+use tokio::task::JoinSet;
+use tokio::time::{Duration, timeout};
+use xkbcommon_rs::{Context, Keymap, KeymapFormat, xkb_keymap::CompileFlags};
+use zbus::{Connection, names::OwnedInterfaceName};
 
-use super::{get_sorted_handlers, CaptureManager, DistanceCalculator};
+pub struct FlatscreenInputPlugin;
+
+impl Plugin for FlatscreenInputPlugin {
+	fn build(&self, app: &mut App) {
+		app.add_systems(Startup, setup);
+		// yes the input method will be delayed by one frame, its only for debugging anyways
+		app.add_systems(Update, update_pointer);
+	}
+}
+
+#[derive(Component)]
+#[require(Camera3d)]
+pub struct FlatscreenCam;
+
+fn setup(mut cmds: Commands) {
+	let Ok(pointer) =
+		MousePointer::new().inspect_err(|err| error!("unable to create mouse pointer: {err}"))
+	else {
+		return;
+	};
+	cmds.spawn((FlatscreenCam, Name::new("Flatscreen Camera")));
+	cmds.insert_resource(pointer);
+}
+
+fn update_pointer(
+	window: Single<(&Window), With<PrimaryWindow>>,
+	mut cam: Single<(&Camera, &GlobalTransform, &mut Transform), With<FlatscreenCam>>,
+	mut pointer: ResMut<MousePointer>,
+	connection: Res<DbusConnection>,
+	object_registry: Res<ObjectRegistryRes>,
+	mouse_buttons: Res<ButtonInput<MouseButton>>,
+	keyboard_buttons: Res<ButtonInput<KeyCode>>,
+	mut scroll: EventReader<MouseWheel>,
+	mut motion: EventReader<MouseMotion>,
+	mut keyboard_input_events: EventReader<KeyboardInput>,
+	time: Res<Time>,
+) {
+	let (cam, cam_transform, mut cam_local_transform) = cam.into_inner();
+	if keyboard_buttons.pressed(KeyCode::ShiftLeft) && mouse_buttons.pressed(MouseButton::Right) {
+		let (mut yaw, mut pitch, _) = cam_local_transform.rotation.to_euler(EulerRot::YXZ);
+
+		for e in motion.read() {
+			let scale = -0.003;
+			pitch += e.delta.y * scale;
+			yaw += e.delta.x * scale;
+		}
+
+		cam_local_transform.rotation = Quat::from_rotation_y(yaw) * Quat::from_rotation_x(pitch);
+
+		let mut move_vec = Vec3::ZERO;
+		move_vec.x += keyboard_buttons.pressed(KeyCode::KeyD) as u32 as f32;
+		move_vec.x -= keyboard_buttons.pressed(KeyCode::KeyA) as u32 as f32;
+		move_vec.z += keyboard_buttons.pressed(KeyCode::KeyS) as u32 as f32;
+		move_vec.z -= keyboard_buttons.pressed(KeyCode::KeyW) as u32 as f32;
+		move_vec.y += keyboard_buttons.pressed(KeyCode::KeyE) as u32 as f32;
+		move_vec.y -= keyboard_buttons.pressed(KeyCode::KeyQ) as u32 as f32;
+
+		let move_vec = cam_local_transform.rotation * move_vec.normalize_or_zero();
+		cam_local_transform.translation += move_vec * time.delta_secs() * 3.0;
+
+		return;
+	}
+	let Some(ray) = window
+		.cursor_position()
+		.and_then(|pos| get_viewport_pos(pos, cam))
+		.and_then(|pos| cam.viewport_to_world(cam_transform, pos).ok())
+	else {
+		return;
+	};
+	pointer.update(
+		&connection,
+		&object_registry,
+		ray,
+		&mouse_buttons,
+		&keyboard_buttons,
+		scroll,
+		keyboard_input_events,
+	);
+}
+
+fn get_viewport_pos(logical_pos: Vec2, cam: &Camera) -> Option<Vec2> {
+	if let Some(viewport_rect) = cam.logical_viewport_rect() {
+		if !viewport_rect.contains(logical_pos) {
+			return None;
+		}
+		Some(logical_pos - viewport_rect.min)
+	} else {
+		Some(logical_pos)
+	}
+}
 
 #[derive(Debug, Deserialize, Serialize)]
 struct MouseEvent {
@@ -46,15 +149,17 @@ impl Default for MouseEvent {
 	}
 }
 
-#[derive(Debug, Clone, Deserialize, Serialize, Default)]
-pub struct KeyboardEvent {
-	pub keyboard: (),
-	pub xkbv1: (),
-	pub keymap_id: u64,
-	pub keys: Vec<i32>,
+#[zbus::proxy(
+	interface = "org.stardustxr.XKBv1",
+	default_service = "org.stardustxr.XKBv1"
+)]
+trait KeyboardHandler {
+	async fn keymap(&self, keymap_id: u64) -> zbus::Result<()>;
+	async fn key_state(&self, key: u32, pressed: bool) -> zbus::Result<()>;
+	async fn reset(&self) -> zbus::Result<()>;
 }
 
-#[allow(unused)]
+#[derive(Resource)]
 pub struct MousePointer {
 	node: OwnedNode,
 	keymap: DefaultKey,
@@ -62,8 +167,6 @@ pub struct MousePointer {
 	pointer: Arc<InputMethod>,
 	capture_manager: CaptureManager,
 	mouse_datamap: MouseEvent,
-	keyboard_datamap: KeyboardEvent,
-	keyboard_sender: Arc<PulseSender>,
 }
 impl MousePointer {
 	pub fn new() -> Result<Self> {
@@ -83,55 +186,68 @@ impl MousePointer {
 				.unwrap(),
 		);
 
-		let keyboard_sender = PulseSender::add_to(
-			&node.0,
-			Datamap::from_typed(KeyboardEvent::default()).unwrap(),
-		)
-		.unwrap();
-
 		Ok(MousePointer {
 			node,
 			spatial,
 			pointer,
 			capture_manager: CaptureManager::default(),
 			mouse_datamap: Default::default(),
-			keyboard_datamap: KeyboardEvent {
-				keyboard: (),
-				xkbv1: (),
-				keymap_id: keymap.data().as_ffi(),
-				keys: vec![],
-			},
 			keymap,
-			keyboard_sender,
 		})
 	}
-	pub fn update(&mut self) {
-		let mouse = Input::get_mouse();
+	pub fn update(
+		&mut self,
+		dbus_connection: &Connection,
+		object_registry: &ObjectRegistry,
+		ray: Ray3d,
+		mouse_buttons: &ButtonInput<MouseButton>,
+		keyboard_buttons: &ButtonInput<KeyCode>,
+		mut scroll: EventReader<MouseWheel>,
+		mut keyboard_input_events: EventReader<KeyboardInput>,
+	) {
+		let mut discrete = Vec2::ZERO;
+		let mut continuous = Vec2::ZERO;
+		for e in scroll.read() {
+			match e.unit {
+				bevy::input::mouse::MouseScrollUnit::Line => {
+					discrete.x += e.x;
+					discrete.y += e.y;
+				}
+				bevy::input::mouse::MouseScrollUnit::Pixel => {
+					continuous.x += e.x;
+					continuous.y += e.y;
+				}
+			}
+		}
 
-		let ray = mouse.get_ray();
 		self.spatial.set_local_transform(
-			Mat4::look_to_rh(
-				Vec3::from(ray.position),
-				Vec3::from(ray.direction),
-				vec3(0.0, 1.0, 0.0),
-			)
-			.inverse(),
+			Mat4::look_to_rh(ray.origin, Vec3::from(ray.direction), Vec3::Y).inverse(),
 		);
 		{
 			// Set pointer input datamap
 			self.mouse_datamap = MouseEvent {
-				select: Input::key(Key::MouseLeft).is_active() as u32 as f32,
-				middle: Input::key(Key::MouseCenter).is_active() as u32 as f32,
-				context: Input::key(Key::MouseRight).is_active() as u32 as f32,
-				grab: (Input::key(Key::Backtick).is_active() && Input::key(Key::Shift).is_active()) as u32 as f32, // Was Mouse 5
-				scroll_continuous: [0.0, mouse.scroll_change / 120.0].into(),
-				scroll_discrete: [0.0, mouse.scroll_change / 120.0].into(),
-				raw_input_events: vec![],
+				select: mouse_buttons.pressed(MouseButton::Left) as u32 as f32,
+				middle: mouse_buttons.pressed(MouseButton::Middle) as u32 as f32,
+				context: mouse_buttons.pressed(MouseButton::Right) as u32 as f32,
+				grab: mouse_buttons.pressed(MouseButton::Right) as u32 as f32, // Was Mouse 5
+				scroll_continuous: continuous.into(),
+				scroll_discrete: discrete.into(),
+				raw_input_events: mouse_buttons
+					.get_pressed()
+					.map(|button| match button {
+						MouseButton::Left => input_event_codes::BTN_LEFT!(),
+						MouseButton::Right => input_event_codes::BTN_RIGHT!(),
+						MouseButton::Middle => input_event_codes::BTN_MIDDLE!(),
+						MouseButton::Back => input_event_codes::BTN_BACK!(),
+						MouseButton::Forward => input_event_codes::BTN_FORWARD!(),
+						MouseButton::Other(b) => *b as u32,
+					})
+					.collect(),
 			};
 			*self.pointer.datamap.lock() = Datamap::from_typed(&self.mouse_datamap).unwrap();
 		}
 		self.target_pointer_input();
-		self.send_keyboard_input();
+		self.send_keyboard_input(dbus_connection, object_registry, keyboard_input_events);
 	}
 	fn target_pointer_input(&mut self) {
 		let distance_calculator: DistanceCalculator = |space, data, field| {
@@ -150,125 +266,177 @@ impl MousePointer {
 			.set_new_capture(&self.pointer, distance_calculator);
 		self.capture_manager.apply_capture(&self.pointer);
 
-		if self.capture_manager.capture.is_some() {
+		if self.capture_manager.capture.upgrade().is_some() {
 			return;
 		}
 
-		let sorted_handlers = get_sorted_handlers(&self.pointer, distance_calculator);
-		self.pointer.set_handler_order(sorted_handlers.iter());
+		let mut handlers = get_sorted_handlers(&self.pointer, distance_calculator);
+		let first_distance = handlers
+			.first()
+			.map(|(_, distance)| *distance)
+			.unwrap_or(f32::NEG_INFINITY);
+
+		self.pointer.set_handler_order(
+			handlers
+				.iter()
+				.filter(|(handler, distance)| (distance - first_distance).abs() <= 0.001)
+				.map(|(handler, _)| handler),
+		);
 	}
 
-	fn send_keyboard_input(&mut self) {
-		let rx = PULSE_RECEIVER_REGISTRY
-			.get_valid_contents()
-			.into_iter()
-			.filter(|rx| mask_matches(&rx.mask, &self.keyboard_sender.mask))
-			.map(|rx| {
-				let result = rx.field.ray_march(Ray {
+	pub fn send_keyboard_input(
+		&mut self,
+		dbus_connection: &Connection,
+		object_registry: &ObjectRegistry,
+		mut keyboard_input_events: EventReader<KeyboardInput>,
+	) {
+		let keyboard_handlers = object_registry.get_objects("org.stardustxr.XKBv1");
+		let events = keyboard_input_events
+			.read()
+			.filter_map(|e| Some((map_key(e.key_code)?, e.state)))
+			.collect::<Vec<_>>();
+		// Spawn async task to handle keyboard input
+		tokio::spawn({
+			let keyboard_handlers = keyboard_handlers.clone();
+			let spatial = self.spatial.clone();
+			let keymap_id = self.keymap.data().as_ffi();
+			let dbus_connection = dbus_connection.clone();
+
+			async move {
+				let mut closest_handler = None;
+				let mut closest_distance = f32::MAX;
+
+				let mut join_set = JoinSet::new();
+				for handler in &keyboard_handlers {
+					let handler = handler.clone();
+					let dbus_connection = dbus_connection.clone();
+					join_set.spawn(async move {
+						// TODO: refactor the whole thing so picking the keyboardhandler to send input to is separate from sending
+						timeout(Duration::from_millis(10), async {
+							let field_ref = handler
+								.to_typed_proxy::<FieldRefProxy>(&dbus_connection)
+								.await
+								.ok()?;
+							let uid = field_ref.uid().await.ok()?;
+							Some((handler, uid))
+						})
+						.await
+						.ok()
+						.flatten()
+					});
+				}
+				while let Some(Ok(Some((handler, field_ref_id)))) = join_set.join_next().await {
+					let exported_fields = EXPORTED_FIELDS.lock();
+					let Some(field_ref_node) =
+						exported_fields.get(&field_ref_id).and_then(|f| f.upgrade())
+					else {
+						println!("didn't find a thing :(");
+						continue;
+					};
+					// println!("still sendin stuff :)");
+					let Ok(field_ref) = field_ref_node.get_aspect::<Field>() else {
+						continue;
+					};
+					drop(exported_fields);
+
+					let result = field_ref.ray_march(Ray {
 						origin: vec3(0.0, 0.0, 0.0),
 						direction: vec3(0.0, 0.0, -1.0),
-					space: self.spatial.clone(),
+						space: spatial.clone(),
 					});
-				(rx, result)
-			})
-			.filter(|(_rx, result)| {
-				result.deepest_point_distance > 0.0 && result.min_distance < 0.05
-			})
-			.reduce(|(rx_a, result_a), (rx_b, result_b)| {
-				if result_a.deepest_point_distance < result_b.deepest_point_distance {
-					(rx_a, result_a)
-				} else {
-					(rx_b, result_b)
-				}
-			})
-			.map(|(rx, _)| rx);
 
-		if let Some(rx) = rx {
-			let keys = (8_u32..254)
-				.map(|i| unsafe { std::mem::transmute(i) })
-				.filter_map(|k| Some((map_key(k)?, Input::key(k))))
-				.filter_map(|(i, k)| {
-					if k.is_just_active() {
-						Some(i as i32)
-					} else if k.is_just_inactive() {
-						Some(-(i as i32))
-					} else {
-						None
-					}
-				})
-				.collect();
-
-			self.keyboard_datamap.keys = keys;
-			if !self.keyboard_datamap.keys.is_empty() {
-				pulse_receiver_client::data(
-					&rx.node.upgrade().unwrap(),
-					&self.node.0,
-					&Datamap::from_typed(&self.keyboard_datamap).unwrap(),
-				)
-				.unwrap();
-			}
-		}
+					if result.deepest_point_distance > 0.0
+						&& result.min_distance < 0.05
+						&& result.deepest_point_distance < closest_distance
+					{
+						closest_distance = result.deepest_point_distance;
+						closest_handler = Some(handler);
 					}
 				}
 
-fn map_key(key: Key) -> Option<u32> {
+				let Some(handler) = closest_handler else {
+					return;
+				};
+				let Ok(keyboard_handler) = handler
+					.to_typed_proxy::<KeyboardHandlerProxy>(&dbus_connection)
+					.await
+				else {
+					return;
+				};
+
+				// Register keymap first
+				let _ = keyboard_handler.keymap(keymap_id).await;
+
+				// Send key states
+				for (key, state) in events.iter() {
+					let pressed = matches!(state, ButtonState::Pressed);
+					let _ = keyboard_handler.key_state(key + 8, pressed).await;
+				}
+			}
+		});
+	}
+}
+
+fn map_key(key: KeyCode) -> Option<u32> {
+	use KeyCode as Key;
 	match key {
+		Key::Unidentified(NativeKeyCode::Xkb(code)) => Some(code),
 		Key::Backspace => Some(input_event_codes::KEY_BACKSPACE!()),
 		Key::Tab => Some(input_event_codes::KEY_TAB!()),
-		Key::Return => Some(input_event_codes::KEY_ENTER!()),
-		Key::Shift => Some(input_event_codes::KEY_LEFTSHIFT!()),
-		Key::Ctrl => Some(input_event_codes::KEY_LEFTCTRL!()),
-		Key::Alt => Some(input_event_codes::KEY_LEFTALT!()),
+		Key::Enter => Some(input_event_codes::KEY_ENTER!()),
+		Key::ShiftLeft => Some(input_event_codes::KEY_LEFTSHIFT!()),
+		Key::ControlLeft => Some(input_event_codes::KEY_LEFTCTRL!()),
+		Key::AltLeft => Some(input_event_codes::KEY_LEFTALT!()),
 		Key::CapsLock => Some(input_event_codes::KEY_CAPSLOCK!()),
-		Key::Esc => Some(input_event_codes::KEY_ESC!()),
+		Key::Escape => Some(input_event_codes::KEY_ESC!()),
 		Key::Space => Some(input_event_codes::KEY_SPACE!()),
 		Key::End => Some(input_event_codes::KEY_END!()),
 		Key::Home => Some(input_event_codes::KEY_HOME!()),
-		Key::Left => Some(input_event_codes::KEY_LEFT!()),
-		Key::Right => Some(input_event_codes::KEY_RIGHT!()),
-		Key::Up => Some(input_event_codes::KEY_UP!()),
-		Key::Down => Some(input_event_codes::KEY_DOWN!()),
+		Key::ArrowLeft => Some(input_event_codes::KEY_LEFT!()),
+		Key::ArrowRight => Some(input_event_codes::KEY_RIGHT!()),
+		Key::ArrowUp => Some(input_event_codes::KEY_UP!()),
+		Key::ArrowDown => Some(input_event_codes::KEY_DOWN!()),
 		Key::PageUp => Some(input_event_codes::KEY_PAGEUP!()),
 		Key::PageDown => Some(input_event_codes::KEY_PAGEDOWN!()),
 		Key::PrintScreen => Some(input_event_codes::KEY_PRINT!()),
-		Key::KeyInsert => Some(input_event_codes::KEY_INSERT!()),
-		Key::Del => Some(input_event_codes::KEY_DELETE!()),
-		Key::Key0 => Some(input_event_codes::KEY_0!()),
-		Key::Key1 => Some(input_event_codes::KEY_1!()),
-		Key::Key2 => Some(input_event_codes::KEY_2!()),
-		Key::Key3 => Some(input_event_codes::KEY_3!()),
-		Key::Key4 => Some(input_event_codes::KEY_4!()),
-		Key::Key5 => Some(input_event_codes::KEY_5!()),
-		Key::Key6 => Some(input_event_codes::KEY_6!()),
-		Key::Key7 => Some(input_event_codes::KEY_7!()),
-		Key::Key8 => Some(input_event_codes::KEY_8!()),
-		Key::Key9 => Some(input_event_codes::KEY_9!()),
-		Key::A => Some(input_event_codes::KEY_A!()),
-		Key::B => Some(input_event_codes::KEY_B!()),
-		Key::C => Some(input_event_codes::KEY_C!()),
-		Key::D => Some(input_event_codes::KEY_D!()),
-		Key::E => Some(input_event_codes::KEY_E!()),
-		Key::F => Some(input_event_codes::KEY_F!()),
-		Key::G => Some(input_event_codes::KEY_G!()),
-		Key::H => Some(input_event_codes::KEY_H!()),
-		Key::I => Some(input_event_codes::KEY_I!()),
-		Key::J => Some(input_event_codes::KEY_J!()),
-		Key::K => Some(input_event_codes::KEY_K!()),
-		Key::L => Some(input_event_codes::KEY_L!()),
-		Key::M => Some(input_event_codes::KEY_M!()),
-		Key::N => Some(input_event_codes::KEY_N!()),
-		Key::O => Some(input_event_codes::KEY_O!()),
-		Key::P => Some(input_event_codes::KEY_P!()),
-		Key::Q => Some(input_event_codes::KEY_Q!()),
-		Key::R => Some(input_event_codes::KEY_R!()),
-		Key::S => Some(input_event_codes::KEY_S!()),
-		Key::T => Some(input_event_codes::KEY_T!()),
-		Key::U => Some(input_event_codes::KEY_U!()),
-		Key::V => Some(input_event_codes::KEY_V!()),
-		Key::W => Some(input_event_codes::KEY_W!()),
-		Key::X => Some(input_event_codes::KEY_X!()),
-		Key::Y => Some(input_event_codes::KEY_Y!()),
-		Key::Z => Some(input_event_codes::KEY_Z!()),
+		Key::Insert => Some(input_event_codes::KEY_INSERT!()),
+		Key::Delete => Some(input_event_codes::KEY_DELETE!()),
+		Key::Digit0 => Some(input_event_codes::KEY_0!()),
+		Key::Digit1 => Some(input_event_codes::KEY_1!()),
+		Key::Digit2 => Some(input_event_codes::KEY_2!()),
+		Key::Digit3 => Some(input_event_codes::KEY_3!()),
+		Key::Digit4 => Some(input_event_codes::KEY_4!()),
+		Key::Digit5 => Some(input_event_codes::KEY_5!()),
+		Key::Digit6 => Some(input_event_codes::KEY_6!()),
+		Key::Digit7 => Some(input_event_codes::KEY_7!()),
+		Key::Digit8 => Some(input_event_codes::KEY_8!()),
+		Key::Digit9 => Some(input_event_codes::KEY_9!()),
+		Key::KeyA => Some(input_event_codes::KEY_A!()),
+		Key::KeyB => Some(input_event_codes::KEY_B!()),
+		Key::KeyC => Some(input_event_codes::KEY_C!()),
+		Key::KeyD => Some(input_event_codes::KEY_D!()),
+		Key::KeyE => Some(input_event_codes::KEY_E!()),
+		Key::KeyF => Some(input_event_codes::KEY_F!()),
+		Key::KeyG => Some(input_event_codes::KEY_G!()),
+		Key::KeyH => Some(input_event_codes::KEY_H!()),
+		Key::KeyI => Some(input_event_codes::KEY_I!()),
+		Key::KeyJ => Some(input_event_codes::KEY_J!()),
+		Key::KeyK => Some(input_event_codes::KEY_K!()),
+		Key::KeyL => Some(input_event_codes::KEY_L!()),
+		Key::KeyM => Some(input_event_codes::KEY_M!()),
+		Key::KeyN => Some(input_event_codes::KEY_N!()),
+		Key::KeyO => Some(input_event_codes::KEY_O!()),
+		Key::KeyP => Some(input_event_codes::KEY_P!()),
+		Key::KeyQ => Some(input_event_codes::KEY_Q!()),
+		Key::KeyR => Some(input_event_codes::KEY_R!()),
+		Key::KeyS => Some(input_event_codes::KEY_S!()),
+		Key::KeyT => Some(input_event_codes::KEY_T!()),
+		Key::KeyU => Some(input_event_codes::KEY_U!()),
+		Key::KeyV => Some(input_event_codes::KEY_V!()),
+		Key::KeyW => Some(input_event_codes::KEY_W!()),
+		Key::KeyX => Some(input_event_codes::KEY_X!()),
+		Key::KeyY => Some(input_event_codes::KEY_Y!()),
+		Key::KeyZ => Some(input_event_codes::KEY_Z!()),
 		Key::Numpad0 => Some(input_event_codes::KEY_NUMERIC_0!()),
 		Key::Numpad1 => Some(input_event_codes::KEY_NUMERIC_1!()),
 		Key::Numpad2 => Some(input_event_codes::KEY_NUMERIC_2!()),
@@ -293,22 +461,22 @@ fn map_key(key: Key) -> Option<u32> {
 		Key::F12 => Some(input_event_codes::KEY_F12!()),
 		Key::Comma => Some(input_event_codes::KEY_COMMA!()),
 		Key::Period => Some(input_event_codes::KEY_DOT!()),
-		Key::SlashFwd => Some(input_event_codes::KEY_SLASH!()),
-		Key::SlashBack => Some(input_event_codes::KEY_BACKSLASH!()),
+		Key::Slash => Some(input_event_codes::KEY_SLASH!()),
+		Key::Backslash => Some(input_event_codes::KEY_BACKSLASH!()),
 		Key::Semicolon => Some(input_event_codes::KEY_SEMICOLON!()),
-		Key::Apostrophe => Some(input_event_codes::KEY_APOSTROPHE!()),
-		Key::BracketOpen => Some(input_event_codes::KEY_LEFTBRACE!()),
-		Key::BracketClose => Some(input_event_codes::KEY_RIGHTBRACE!()),
+		Key::Quote => Some(input_event_codes::KEY_APOSTROPHE!()),
+		Key::BracketLeft => Some(input_event_codes::KEY_LEFTBRACE!()),
+		Key::BracketRight => Some(input_event_codes::KEY_RIGHTBRACE!()),
 		Key::Minus => Some(input_event_codes::KEY_MINUS!()),
-		Key::Equals => Some(input_event_codes::KEY_EQUAL!()),
-		Key::Backtick => Some(input_event_codes::KEY_GRAVE!()),
-		Key::LCmd => Some(input_event_codes::KEY_LEFTMETA!()),
-		Key::RCmd => Some(input_event_codes::KEY_RIGHTMETA!()),
-		Key::Multiply => Some(input_event_codes::KEY_NUMERIC_STAR!()),
-		Key::Add => Some(input_event_codes::KEY_KPPLUS!()),
-		Key::Subtract => Some(input_event_codes::KEY_MINUS!()),
-		Key::Decimal => Some(input_event_codes::KEY_DOT!()),
-		Key::Divide => Some(input_event_codes::KEY_SLASH!()),
+		Key::Equal => Some(input_event_codes::KEY_EQUAL!()),
+		Key::Backquote => Some(input_event_codes::KEY_GRAVE!()),
+		Key::SuperLeft => Some(input_event_codes::KEY_LEFTMETA!()),
+		Key::SuperRight => Some(input_event_codes::KEY_RIGHTMETA!()),
+		Key::NumpadMultiply => Some(input_event_codes::KEY_NUMERIC_STAR!()),
+		Key::NumpadAdd => Some(input_event_codes::KEY_KPPLUS!()),
+		Key::NumpadSubtract => Some(input_event_codes::KEY_MINUS!()),
+		Key::NumpadDecimal => Some(input_event_codes::KEY_DOT!()),
+		Key::NumpadDivide => Some(input_event_codes::KEY_SLASH!()),
 		_ => None,
 	}
 }

src/objects/input/oxr_controller.rs

@@ -0,0 +1,414 @@
+use super::{CaptureManager, get_sorted_handlers};
+use crate::{
+	DbusConnection, PreFrameWait,
+	core::client::INTERNAL_CLIENT,
+	nodes::{
+		Node, OwnedNode,
+		drawable::{
+			MaterialParameter,
+			model::{Model, ModelPart},
+		},
+		fields::{Field, FieldTrait},
+		input::{INPUT_HANDLER_REGISTRY, InputDataType, InputHandler, InputMethod, Tip},
+		spatial::Spatial,
+	},
+	objects::{ObjectHandle, SpatialRef, Tracked},
+};
+use bevy::{asset::Handle, ecs::resource::Resource};
+use bevy::{math::Affine3, prelude::*};
+use bevy_mod_openxr::{
+	action_binding::{OxrSendActionBindings, OxrSuggestActionBinding},
+	helper_traits::{ToIsometry3d, ToVec2},
+	resources::{OxrFrameState, OxrInstance},
+	session::OxrSession,
+};
+use bevy_mod_xr::{
+	hands::HandSide,
+	session::{XrPreDestroySession, XrSessionCreated, XrSessionCreatedEvent},
+	spaces::{XrPrimaryReferenceSpace, XrReferenceSpace, XrSpace},
+};
+use color_eyre::eyre::Result;
+use glam::{Affine3A, Mat4, Vec2, Vec3};
+use openxr::{Action, ActiveActionSet, SpaceLocationFlags};
+use serde::{Deserialize, Serialize};
+use stardust_xr::values::{Datamap, ResourceID, color::Rgb};
+use std::{
+	borrow::Cow,
+	fs,
+	path::{Path, PathBuf},
+	str::FromStr,
+	sync::Arc,
+};
+use tracing::instrument;
+use zbus::Connection;
+pub struct ControllerPlugin;
+const CURSOR_MODEL_PATH: &str = "/tmp/stardust_server/models/cursor.glb";
+impl Plugin for ControllerPlugin {
+	fn build(&self, app: &mut App) {
+		let cursor = include_bytes!("cursor.glb");
+		fs::create_dir_all(
+			PathBuf::from_str(CURSOR_MODEL_PATH)
+				.unwrap()
+				.parent()
+				.unwrap(),
+		);
+		fs::write(CURSOR_MODEL_PATH, cursor).expect("can't write tmp cursor model file");
+		app.add_systems(OxrSendActionBindings, suggest_bindings.run_if(run_once));
+		app.add_systems(
+			PostUpdate,
+			create_spaces.run_if(on_event::<XrSessionCreatedEvent>),
+		);
+		app.add_systems(XrPreDestroySession, destroy_spaces);
+		app.add_systems(Startup, setup.run_if(resource_exists::<OxrInstance>));
+		app.add_systems(PreFrameWait, update.run_if(resource_exists::<Controllers>));
+	}
+}
+
+// the api is just slightly nicer when using the bevy_mod_openxr solution okay?
+fn suggest_bindings(
+	instance: Res<OxrInstance>,
+	actions: Res<Actions>,
+	mut suggest: EventWriter<OxrSuggestActionBinding>,
+) {
+	let mut bind_all = |interaction_profile: &'static str,
+	                    bindings: &[(openxr::sys::Action, &[&'static str])]| {
+		for (action, bindings) in bindings {
+			suggest.write(OxrSuggestActionBinding {
+				action: *action,
+				interaction_profile: interaction_profile.into(),
+				bindings: bindings.iter().copied().map(Cow::Borrowed).collect(),
+			});
+		}
+	};
+	bind_all(
+		"/interaction_profiles/oculus/touch_controller",
+		&[
+			(
+				actions.trigger.as_raw(),
+				&[
+					"/user/hand/left/input/trigger/value",
+					"/user/hand/right/input/trigger/value",
+				],
+			),
+			(
+				actions.stick_click.as_raw(),
+				&[
+					"/user/hand/left/input/thumbstick/click",
+					"/user/hand/right/input/thumbstick/click",
+				],
+			),
+			(
+				actions.button.as_raw(),
+				&[
+					"/user/hand/left/input/x/click",
+					"/user/hand/left/input/y/click",
+					"/user/hand/right/input/a/click",
+					"/user/hand/right/input/b/click",
+				],
+			),
+			(
+				actions.grip.as_raw(),
+				&[
+					"/user/hand/left/input/squeeze/value",
+					"/user/hand/right/input/squeeze/value",
+				],
+			),
+			(
+				actions.stick.as_raw(),
+				&[
+					"/user/hand/left/input/thumbstick",
+					"/user/hand/right/input/thumbstick",
+				],
+			),
+			(
+				actions.space.as_raw(),
+				&[
+					"/user/hand/left/input/aim/pose",
+					"/user/hand/right/input/aim/pose",
+				],
+			),
+		],
+	);
+	bind_all(
+		"/interaction_profiles/khr/simple_controller",
+		&[(
+			actions.space.as_raw(),
+			&[
+				"/user/hand/left/input/aim/pose",
+				"/user/hand/right/input/aim/pose",
+			],
+		)],
+	);
+}
+
+fn update(
+	mut controllers: ResMut<Controllers>,
+	actions: Res<Actions>,
+	session: Option<Res<OxrSession>>,
+	ref_space: Option<Res<XrPrimaryReferenceSpace>>,
+	state: Option<Res<OxrFrameState>>,
+) {
+	let (Some(session), Some(state), Some(ref_space)) = (session, state, ref_space) else {
+		controllers.left.set_enabled(false);
+		controllers.right.set_enabled(false);
+		return;
+	};
+	debug_span!("sync actions").in_scope(|| {
+		session
+			.sync_actions(&[ActiveActionSet::new(&actions.set)])
+			.unwrap();
+	});
+	let time = state.predicted_display_time;
+	// stupid bevy gltf loading issue (rotated 180 degrees on the y axis)
+	controllers
+		.left
+		.update(&session, &actions, time, ref_space.0);
+	controllers
+		.right
+		.update(&session, &actions, time, ref_space.0);
+}
+
+fn create_spaces(
+	session: Res<OxrSession>,
+	mut controllers: ResMut<Controllers>,
+	actions: Res<Actions>,
+) {
+	// if we ever need more actions than just these we should fully swith to the
+	// bevy_mod_openxr provided stuff
+	session.attach_action_sets(&[&actions.set]);
+	session
+		.sync_actions(&[ActiveActionSet::new(&actions.set)])
+		.unwrap();
+
+	let instance = session.instance();
+	let left = instance.string_to_path("/user/hand/left").unwrap();
+	let right = instance.string_to_path("/user/hand/right").unwrap();
+	let left = session
+		.create_action_space(&actions.space, left, Isometry3d::IDENTITY)
+		.unwrap();
+	let right = session
+		.create_action_space(&actions.space, right, Isometry3d::IDENTITY)
+		.unwrap();
+	controllers.left.space = Some(left);
+	controllers.right.space = Some(right);
+}
+
+fn destroy_spaces(session: Res<OxrSession>, mut controllers: ResMut<Controllers>) {
+	if let Some(space) = controllers.left.space.take() {
+		session.destroy_space(space);
+	}
+	if let Some(space) = controllers.right.space.take() {
+		session.destroy_space(space);
+	}
+}
+
+fn setup(instance: Res<OxrInstance>, connection: Res<DbusConnection>, mut cmds: Commands) {
+	tokio::task::spawn({
+		let connection = connection.clone();
+		async move {
+			connection
+				.request_name("org.stardustxr.Controllers")
+				.await
+				.unwrap();
+		}
+	});
+	let set = instance
+		.create_action_set("input_method_actions", "Input Method Action Source", 0)
+		.unwrap();
+	let paths = &[
+		instance.string_to_path("/user/hand/left").unwrap(),
+		instance.string_to_path("/user/hand/right").unwrap(),
+	];
+	let actions = Actions {
+		trigger: set.create_action("trigger", "Select", paths).unwrap(),
+		stick_click: set.create_action("stick_click", "Middle", paths).unwrap(),
+		button: set.create_action("face_button", "Context", paths).unwrap(),
+		grip: set.create_action("grip", "Grab", paths).unwrap(),
+		stick: set.create_action("stick", "Scroll", paths).unwrap(),
+		space: set.create_action("pose", "Location", paths).unwrap(),
+		set,
+	};
+	let controllers = Controllers {
+		left: OxrControllerInput::new(&connection, HandSide::Left).unwrap(),
+		right: OxrControllerInput::new(&connection, HandSide::Right).unwrap(),
+	};
+	cmds.insert_resource(controllers);
+	cmds.insert_resource(actions);
+}
+
+#[derive(Default, Debug, Deserialize, Serialize)]
+struct ControllerDatamap {
+	select: f32,
+	middle: f32,
+	context: f32,
+	grab: f32,
+	scroll: Vec2,
+}
+#[derive(Resource)]
+struct Actions {
+	set: openxr::ActionSet,
+	trigger: openxr::Action<f32>,
+	stick_click: openxr::Action<f32>,
+	button: openxr::Action<f32>,
+	grip: openxr::Action<f32>,
+	space: openxr::Action<openxr::Posef>,
+	stick: openxr::Action<openxr::Vector2f>,
+}
+#[derive(Resource)]
+struct Controllers {
+	left: OxrControllerInput,
+	right: OxrControllerInput,
+}
+
+pub struct OxrControllerInput {
+	object_handle: ObjectHandle<SpatialRef>,
+	input: Arc<InputMethod>,
+	side: HandSide,
+	model: Arc<Model>,
+	model_part: Arc<ModelPart>,
+	capture_manager: CaptureManager,
+	datamap: ControllerDatamap,
+	tracked: ObjectHandle<Tracked>,
+	space: Option<XrSpace>,
+}
+impl OxrControllerInput {
+	fn new(connection: &Connection, side: HandSide) -> Result<Self> {
+		let path = "/org/stardustxr/Controller/".to_string()
+			+ match side {
+				HandSide::Left => "left",
+				HandSide::Right => "right",
+			};
+		let (spatial, object_handle) = SpatialRef::create(connection, &path);
+		let tracked = Tracked::new(connection, &path);
+		let tip = InputDataType::Tip(Tip::default());
+		let node = spatial.node().unwrap();
+		node.set_enabled(false);
+		let model = Model::add_to(&node, ResourceID::Direct(CURSOR_MODEL_PATH.into())).unwrap();
+		let model_part = model.get_model_part("Cursor".to_string()).unwrap();
+		let input = InputMethod::add_to(
+			&node,
+			tip,
+			Datamap::from_typed(ControllerDatamap::default())?,
+		)?;
+		Ok(OxrControllerInput {
+			object_handle,
+			input,
+			side,
+			model,
+			model_part,
+			capture_manager: CaptureManager::default(),
+			datamap: Default::default(),
+			tracked,
+			space: None,
+		})
+	}
+	#[instrument(level = "debug", skip(self))]
+	pub fn set_enabled(&self, enabled: bool) {
+		if let Some(node) = self.input.spatial.node() {
+			node.set_enabled(enabled);
+		}
+		tokio::spawn({
+			// this is suboptimal since it probably allocates a fresh string every frame
+			let handle = self.tracked.clone();
+			async move {
+				handle.set_tracked(enabled).await;
+			}
+		});
+	}
+	fn update(
+		&mut self,
+		session: &OxrSession,
+		actions: &Actions,
+		time: openxr::Time,
+		ref_space: XrReferenceSpace,
+	) {
+		let Some(space) = self.space.as_ref() else {
+			return;
+		};
+		let _span = debug_span!("locate space").entered();
+		let Ok(location) = session
+			.locate_space(space, &ref_space, time)
+			.inspect_err(|err| error!("error while locating controller space: {err}"))
+		else {
+			return;
+		};
+		let enabled = location.location_flags.contains(
+			SpaceLocationFlags::POSITION_VALID
+				| SpaceLocationFlags::POSITION_TRACKED
+				| SpaceLocationFlags::ORIENTATION_VALID
+				| SpaceLocationFlags::ORIENTATION_TRACKED,
+		);
+		drop(_span);
+		self.set_enabled(enabled);
+		if enabled {
+			let world_transform = Mat4::from(Affine3A::from(location.pose.to_xr_pose()));
+			self.model_part
+				.set_material_parameter("roughness".to_string(), MaterialParameter::Float(1.0));
+			self.model_part.set_material_parameter(
+				"color".to_string(),
+				MaterialParameter::Color(stardust_xr::values::Color::new(
+					if self.capture_manager.capture.upgrade().is_none() {
+						Rgb::new(1.0, 1.0, 1.0)
+					} else {
+						Rgb::new(0.0, 1.0, 0.75)
+					},
+					1.0,
+				)),
+			);
+			self.input
+				.spatial
+				.set_local_transform(world_transform * Mat4::from_scale(Vec3::splat(0.02)));
+		}
+		let path = session
+			.instance()
+			.string_to_path(match self.side {
+				HandSide::Left => "/user/hand/left",
+				HandSide::Right => "/user/hand/right",
+			})
+			.unwrap();
+		if let Ok(path) = session.current_interaction_profile(path) {
+			if session.instance().path_to_string(path).unwrap()
+				== "/interaction_profiles/khr/simple_controller"
+			{
+				self.set_enabled(false);
+			}
+		}
+		fn get<T: openxr::ActionInput + Default>(
+			session: &OxrSession,
+			path: openxr::Path,
+			action: &Action<T>,
+		) -> T {
+			action
+				.state(session, path)
+				.map(|v| v.current_state)
+				.unwrap_or_default()
+		}
+		let _span = debug_span!("apply datamap").entered();
+		self.datamap = ControllerDatamap {
+			select: get(session, path, &actions.trigger),
+			middle: get(session, path, &actions.stick_click) as u32 as f32,
+			context: get(session, path, &actions.button) as u32 as f32,
+			grab: get(session, path, &actions.grip),
+			scroll: get(session, path, &actions.stick).to_vec2(),
+		};
+		*self.input.datamap.lock() = Datamap::from_typed(&self.datamap).unwrap();
+		drop(_span);
+
+		let distance_calculator = |space: &Arc<Spatial>, _data: &InputDataType, field: &Field| {
+			Some(field.distance(space, [0.0; 3].into()).abs())
+		};
+
+		self.capture_manager.update_capture(&self.input);
+		self.capture_manager
+			.set_new_capture(&self.input, distance_calculator);
+		self.capture_manager.apply_capture(&self.input);
+
+		if self.capture_manager.capture.upgrade().is_some() {
+			return;
+		}
+
+		let sorted_handlers = get_sorted_handlers(&self.input, distance_calculator);
+		self.input
+			.set_handler_order(sorted_handlers.iter().map(|(handler, _)| handler));
+	}
+}

src/objects/input/oxr_hand.rs

@@ -0,0 +1,371 @@
+use crate::core::client::INTERNAL_CLIENT;
+use crate::nodes::OwnedNode;
+use crate::nodes::fields::{Field, FieldTrait};
+use crate::nodes::input::{Finger, INPUT_HANDLER_REGISTRY, InputDataType, InputHandler, Thumb};
+use crate::nodes::{
+	Node,
+	input::{Hand, InputMethod, Joint},
+	spatial::Spatial,
+};
+use crate::objects::{ObjectHandle, SpatialRef, Tracked};
+use crate::{BevyMaterial, DbusConnection, ObjectRegistryRes, PreFrameWait};
+use bevy::prelude::Transform as BevyTransform;
+use bevy::prelude::*;
+use bevy_mod_openxr::helper_traits::{ToQuat, ToVec3};
+use bevy_mod_openxr::resources::OxrFrameState;
+use bevy_mod_openxr::session::OxrSession;
+use bevy_mod_xr::hands::{HandBone, HandSide, XrHandBoneEntities, XrHandBoneRadius};
+use bevy_mod_xr::session::{XrPreDestroySession, XrSessionCreated};
+use bevy_mod_xr::spaces::{XrPrimaryReferenceSpace, XrSpaceLocationFlags};
+use bevy_sk::hand::GRADIENT_TEXTURE_HANDLE;
+use color_eyre::eyre::Result;
+use glam::{Mat4, Quat, Vec3};
+use openxr::{HandJointLocation, SpaceLocationFlags};
+use serde::{Deserialize, Serialize};
+use stardust_xr::values::Datamap;
+use std::sync::Arc;
+use zbus::Connection;
+
+use super::{CaptureManager, get_sorted_handlers};
+
+pub struct HandPlugin;
+impl Plugin for HandPlugin {
+	fn build(&self, app: &mut App) {
+		app.add_systems(PreFrameWait, update_hands.run_if(resource_exists::<Hands>));
+		app.add_systems(XrSessionCreated, create_trackers);
+		app.add_systems(XrPreDestroySession, destroy_trackers);
+		app.add_systems(PostUpdate, update_hand_material);
+		app.add_systems(Startup, setup);
+	}
+}
+fn update_hands(
+	mut hands: ResMut<Hands>,
+	session: Option<Res<OxrSession>>,
+	state: Option<Res<OxrFrameState>>,
+	ref_space: Option<Res<XrPrimaryReferenceSpace>>,
+	mut materials: ResMut<Assets<BevyMaterial>>,
+	mut joint_query: Query<(
+		&mut BevyTransform,
+		&mut XrSpaceLocationFlags,
+		&mut XrHandBoneRadius,
+	)>,
+	joints_query: Query<&XrHandBoneEntities>,
+) {
+	let (Some(session), Some(state), Some(ref_space)) = (session, state, ref_space) else {
+		tokio::task::spawn({
+			let left = hands.left.tracked.clone();
+			let right = hands.right.tracked.clone();
+			async move {
+				left.set_tracked(false);
+				right.set_tracked(false);
+			}
+		});
+		return;
+	};
+	let get_joints = |hand: &mut OxrHandInput| -> Option<openxr::HandJointLocations> {
+		let Some(tracker) = hand.tracker.as_ref() else {
+			hand.input.spatial.node().unwrap().set_enabled(false);
+			let handle = hand.tracked.clone();
+			tokio::task::spawn(async move {
+				handle.set_tracked(false);
+			});
+			return None;
+		};
+		// this won't be correct with pipelined rendering
+		session
+			.locate_hand_joints(tracker, &ref_space, state.predicted_display_time)
+			.inspect_err(|err| error!("Error while locating hand joints"))
+			.ok()
+			.flatten()
+	};
+	let joints_left = get_joints(&mut hands.left);
+	let joints_right = get_joints(&mut hands.right);
+	hands.left.update(joints_left.as_ref(), &mut materials);
+	hands.right.update(joints_right.as_ref(), &mut materials);
+}
+
+fn pinch_between(joint_1: &Joint, joint_2: &Joint) -> f32 {
+	const PINCH_MAX: f32 = 0.11;
+	const PINCH_ACTIVACTION_DISTANCE: f32 = 0.01;
+	let combined_radius = joint_1.radius + joint_2.radius;
+	let pinch_dist =
+		Vec3::from(joint_1.position).distance(Vec3::from(joint_2.position)) - combined_radius;
+	(1.0 - ((pinch_dist - PINCH_ACTIVACTION_DISTANCE) / (PINCH_MAX - PINCH_ACTIVACTION_DISTANCE)))
+		.clamp(0.0, 1.0)
+}
+
+fn create_trackers(session: Res<OxrSession>, mut hands: ResMut<Hands>) {
+	hands.left.tracker = session
+		.create_hand_tracker(openxr::HandEXT::LEFT)
+		.inspect_err(|err| error!("failed to create left hand tracker"))
+		.ok();
+	hands.right.tracker = session
+		.create_hand_tracker(openxr::HandEXT::RIGHT)
+		.inspect_err(|err| error!("failed to create right hand tracker"))
+		.ok();
+}
+fn destroy_trackers(mut hands: ResMut<Hands>) {
+	hands.left.tracker.take();
+	hands.right.tracker.take();
+}
+#[derive(Component)]
+struct CorrectHandMaterial;
+fn update_hand_material(
+	query: Query<
+		(Entity, &HandSide),
+		(
+			With<XrHandBoneEntities>,
+			With<MeshMaterial3d<BevyMaterial>>,
+			Without<CorrectHandMaterial>,
+		),
+	>,
+	mut cmds: Commands,
+	hands: Res<Hands>,
+) {
+	for (entity, side) in &query {
+		let handle = match side {
+			HandSide::Left => hands.left.material.clone(),
+			HandSide::Right => hands.right.material.clone(),
+		};
+		cmds.entity(entity)
+			.insert(MeshMaterial3d(handle))
+			.insert(CorrectHandMaterial);
+	}
+}
+
+fn setup(
+	connection: Res<DbusConnection>,
+	mut cmds: Commands,
+	mut materials: ResMut<Assets<BevyMaterial>>,
+) {
+	tokio::task::spawn({
+		let connection = connection.clone();
+		async move {
+			connection
+				.request_name("org.stardustxr.Hands")
+				.await
+				.unwrap();
+		}
+	});
+	cmds.insert_resource(Hands {
+		left: OxrHandInput::new(&connection, HandSide::Left, &mut materials).unwrap(),
+		right: OxrHandInput::new(&connection, HandSide::Right, &mut materials).unwrap(),
+	});
+}
+
+fn convert_joint(joint: HandJointLocation) -> Joint {
+	Joint {
+		position: joint.pose.position.to_vec3().into(),
+		rotation: joint.pose.orientation.to_quat().into(),
+		radius: joint.radius,
+		distance: 0.0,
+	}
+}
+
+#[derive(Resource)]
+struct Hands {
+	left: OxrHandInput,
+	right: OxrHandInput,
+}
+
+#[derive(Default, Deserialize, Serialize)]
+struct HandDatamap {
+	pinch_strength: f32,
+	grab_strength: f32,
+}
+
+pub struct OxrHandInput {
+	_node: OwnedNode,
+	palm_spatial: Arc<Spatial>,
+	palm_object: ObjectHandle<SpatialRef>,
+	side: HandSide,
+	input: Arc<InputMethod>,
+	capture_manager: CaptureManager,
+	datamap: HandDatamap,
+	tracked: ObjectHandle<Tracked>,
+	tracker: Option<openxr::HandTracker>,
+	captured: bool,
+	material: Handle<BevyMaterial>,
+}
+impl OxrHandInput {
+	pub fn new(
+		connection: &Connection,
+		side: HandSide,
+		materials: &mut Assets<BevyMaterial>,
+	) -> Result<Self> {
+		let (palm_spatial, palm_object) = SpatialRef::create(
+			connection,
+			&("/org/stardustxr/Hand/".to_string()
+				+ match side {
+					HandSide::Left => "left",
+					HandSide::Right => "right",
+				} + "/palm"),
+		);
+		let tracked = Tracked::new(
+			connection,
+			&("/org/stardustxr/Hand/".to_string()
+				+ match side {
+					HandSide::Left => "left",
+					HandSide::Right => "right",
+				}),
+		);
+		let node = Node::generate(&INTERNAL_CLIENT, false).add_to_scenegraph_owned()?;
+		Spatial::add_to(&node.0, None, Mat4::IDENTITY, false);
+		let hand = InputDataType::Hand(Hand {
+			right: matches!(side, HandSide::Right),
+			..Default::default()
+		});
+		let datamap = Datamap::from_typed(HandDatamap::default())?;
+		let input = InputMethod::add_to(&node.0, hand, datamap)?;
+
+		let material = materials.add(BevyMaterial {
+			base_color: Srgba::new(1.0, 1.0, 1.0, 1.0).into(),
+			alpha_mode: AlphaMode::Blend,
+			base_color_texture: Some(GRADIENT_TEXTURE_HANDLE),
+			perceptual_roughness: 1.0,
+			..default()
+		});
+		Ok(OxrHandInput {
+			_node: node,
+			palm_spatial,
+			palm_object,
+			side,
+			input,
+			tracked,
+			capture_manager: CaptureManager::default(),
+			datamap: Default::default(),
+			tracker: None,
+			material,
+			captured: false,
+		})
+	}
+	pub fn set_enabled(&self, enabled: bool) {
+		if let Some(node) = self.input.spatial.node() {
+			node.set_enabled(enabled);
+		}
+		tokio::spawn({
+			// this is suboptimal since it probably allocates a fresh string every frame
+			let handle = self.tracked.clone();
+			async move {
+				handle.set_tracked(enabled).await;
+			}
+		});
+	}
+	fn update(
+		&mut self,
+		joints: Option<&openxr::HandJointLocations>,
+		materials: &mut ResMut<Assets<BevyMaterial>>,
+	) {
+		// TODO: use the hand data source ext
+		let real_hand = true;
+		let input_node = self.input.spatial.node().unwrap();
+		let is_tracked = real_hand
+			&& joints.is_some_and(|v| {
+				v.iter().all(|v| {
+					v.location_flags.contains(
+						SpaceLocationFlags::POSITION_VALID | SpaceLocationFlags::POSITION_TRACKED,
+					) || v.location_flags.contains(
+						SpaceLocationFlags::ORIENTATION_VALID
+							| SpaceLocationFlags::ORIENTATION_TRACKED,
+					)
+				})
+			});
+		self.set_enabled(is_tracked);
+		if is_tracked {
+			// cannot ever crash, is_tracked is only true of joints is some
+			let joints = joints.unwrap();
+			let new_hand = Hand {
+				right: matches!(self.side, HandSide::Right),
+				thumb: Thumb {
+					tip: convert_joint(joints[HandBone::ThumbTip as usize]),
+					distal: convert_joint(joints[HandBone::ThumbDistal as usize]),
+					proximal: convert_joint(joints[HandBone::ThumbProximal as usize]),
+					metacarpal: convert_joint(joints[HandBone::ThumbMetacarpal as usize]),
+				},
+				index: Finger {
+					tip: convert_joint(joints[HandBone::IndexTip as usize]),
+					distal: convert_joint(joints[HandBone::IndexDistal as usize]),
+					intermediate: convert_joint(joints[HandBone::IndexIntermediate as usize]),
+					proximal: convert_joint(joints[HandBone::IndexProximal as usize]),
+					metacarpal: convert_joint(joints[HandBone::IndexMetacarpal as usize]),
+				},
+				middle: Finger {
+					tip: convert_joint(joints[HandBone::MiddleTip as usize]),
+					distal: convert_joint(joints[HandBone::MiddleDistal as usize]),
+					intermediate: convert_joint(joints[HandBone::MiddleIntermediate as usize]),
+					proximal: convert_joint(joints[HandBone::MiddleProximal as usize]),
+					metacarpal: convert_joint(joints[HandBone::MiddleMetacarpal as usize]),
+				},
+				ring: Finger {
+					tip: convert_joint(joints[HandBone::RingTip as usize]),
+					distal: convert_joint(joints[HandBone::RingDistal as usize]),
+					intermediate: convert_joint(joints[HandBone::RingIntermediate as usize]),
+					proximal: convert_joint(joints[HandBone::RingProximal as usize]),
+					metacarpal: convert_joint(joints[HandBone::RingMetacarpal as usize]),
+				},
+				little: Finger {
+					tip: convert_joint(joints[HandBone::LittleTip as usize]),
+					distal: convert_joint(joints[HandBone::LittleDistal as usize]),
+					intermediate: convert_joint(joints[HandBone::LittleIntermediate as usize]),
+					proximal: convert_joint(joints[HandBone::LittleProximal as usize]),
+					metacarpal: convert_joint(joints[HandBone::LittleMetacarpal as usize]),
+				},
+				palm: convert_joint(joints[HandBone::Palm as usize]),
+				wrist: convert_joint(joints[HandBone::Wrist as usize]),
+				elbow: None,
+			};
+			self.palm_spatial
+				.set_local_transform(Mat4::from_rotation_translation(
+					new_hand.palm.rotation.into(),
+					new_hand.palm.position.into(),
+				));
+
+			self.datamap.pinch_strength = pinch_between(&new_hand.thumb.tip, &new_hand.index.tip);
+			// this is how stereokit calculates grab
+			self.datamap.grab_strength =
+				pinch_between(&new_hand.ring.tip, &new_hand.ring.metacarpal);
+
+			*self.input.data.lock() = InputDataType::Hand(new_hand);
+			*self.input.datamap.lock() = Datamap::from_typed(&self.datamap).unwrap();
+			let captured = self.capture_manager.capture.upgrade().is_some();
+			if captured && !self.captured {
+				materials.get_mut(&self.material).unwrap().base_color =
+					Srgba::rgb(0., 1., 0.75).into();
+			} else if self.captured && !captured {
+				materials.get_mut(&self.material).unwrap().base_color =
+					Srgba::rgb(1., 1.0, 1.0).into();
+			}
+			self.captured = captured;
+		}
+
+		let distance_calculator = |space: &Arc<Spatial>, data: &InputDataType, field: &Field| {
+			let InputDataType::Hand(hand) = data else {
+				return None;
+			};
+			let thumb_tip_distance = field.distance(space, hand.thumb.tip.position.into());
+			let index_tip_distance = field.distance(space, hand.index.tip.position.into());
+			let middle_tip_distance = field.distance(space, hand.middle.tip.position.into());
+			let ring_tip_distance = field.distance(space, hand.ring.tip.position.into());
+
+			Some(
+				(thumb_tip_distance * 0.3)
+					+ (index_tip_distance * 0.4)
+					+ (middle_tip_distance * 0.15)
+					+ (ring_tip_distance * 0.15),
+			)
+		};
+
+		self.capture_manager.update_capture(&self.input);
+		self.capture_manager
+			.set_new_capture(&self.input, distance_calculator);
+		self.capture_manager.apply_capture(&self.input);
+
+		if self.capture_manager.capture.upgrade().is_some() {
+			return;
+		}
+
+		let sorted_handlers = get_sorted_handlers(&self.input, distance_calculator);
+		self.input
+			.set_handler_order(sorted_handlers.iter().map(|(handler, _)| handler));
+	}
+}

src/objects/input/sk_controller.rs

@@ -1,128 +0,0 @@
-use super::{get_sorted_handlers, CaptureManager};
-use crate::{
-	core::client::INTERNAL_CLIENT,
-	nodes::{
-		fields::{Field, FieldTrait},
-		input::{InputDataType, InputHandler, InputMethod, Tip, INPUT_HANDLER_REGISTRY},
-		spatial::Spatial,
-		Node, OwnedNode,
-	},
-	objects::{ObjectHandle, SpatialRef},
-};
-use color_eyre::eyre::Result;
-use glam::{Mat4, Vec2, Vec3};
-use serde::{Deserialize, Serialize};
-use stardust_xr::values::Datamap;
-use std::sync::Arc;
-use stereokit_rust::{
-	material::Material,
-	model::Model,
-	sk::MainThreadToken,
-	system::{Handed, Input},
-	util::Color128,
-};
-use zbus::Connection;
-
-#[derive(Default, Debug, Deserialize, Serialize)]
-struct ControllerDatamap {
-	select: f32,
-	middle: f32,
-	context: f32,
-	grab: f32,
-	scroll: Vec2,
-}
-
-pub struct SkController {
-	object_handle: ObjectHandle<SpatialRef>,
-	input: Arc<InputMethod>,
-	handed: Handed,
-	model: Model,
-	material: Material,
-	capture_manager: CaptureManager,
-	datamap: ControllerDatamap,
-}
-impl SkController {
-	pub fn new(connection: &Connection, handed: Handed) -> Result<Self> {
-		let (spatial, object_handle) = SpatialRef::create(
-			connection,
-			&("/org/stardustxr/Controller/".to_string()
-				+ match handed {
-					Handed::Left => "left",
-					_ => "right",
-				}),
-		);
-		let model = Model::copy(Model::from_memory(
-			"cursor.glb",
-			include_bytes!("cursor.glb"),
-			None,
-		)?);
-		let model_nodes = model.get_nodes();
-		let mut model_node = model_nodes.visuals().next().unwrap();
-		let material = Material::copy(&model_node.get_material().unwrap());
-		model_node.material(&material);
-		let tip = InputDataType::Tip(Tip::default());
-		let input = InputMethod::add_to(
-			&spatial.node().unwrap(),
-			tip,
-			Datamap::from_typed(ControllerDatamap::default())?,
-		)?;
-		Ok(SkController {
-			object_handle,
-			input,
-			handed,
-			model,
-			material,
-			capture_manager: CaptureManager::default(),
-			datamap: Default::default(),
-		})
-	}
-	pub fn update(&mut self, token: &MainThreadToken) {
-		let controller = Input::controller(self.handed);
-		let input_node = self.input.spatial.node().unwrap();
-		input_node.set_enabled(controller.tracked.is_active());
-		if input_node.enabled() {
-			let world_transform = Mat4::from_rotation_translation(
-				controller.aim.orientation.into(),
-				controller.aim.position.into(),
-			);
-			self.material
-				.color_tint(if self.capture_manager.capture.is_none() {
-					Color128::new_rgb(1.0, 1.0, 1.0)
-				} else {
-					Color128::new_rgb(0.0, 1.0, 0.75)
-				});
-			self.model.draw(
-				token,
-				world_transform * Mat4::from_scale(Vec3::ONE * 0.02),
-				None,
-				None,
-			);
-			self.input.spatial.set_local_transform(world_transform);
-		}
-
-		self.datamap = ControllerDatamap {
-			select: controller.trigger,
-			middle: controller.stick_click.is_active() as u32 as f32,
-			context: controller.is_x2_pressed() as u32 as f32,
-			grab: controller.grip,
-			scroll: controller.stick.into(),
-		};
-		*self.input.datamap.lock() = Datamap::from_typed(&self.datamap).unwrap();
-
-		let distance_calculator = |space: &Arc<Spatial>, _data: &InputDataType, field: &Field| {
-			Some(field.distance(space, [0.0; 3].into()).abs())
-		};
-
-		self.capture_manager.update_capture(&self.input);
-		self.capture_manager
-			.set_new_capture(&self.input, distance_calculator);
-		self.capture_manager.apply_capture(&self.input);
-
-		if self.capture_manager.capture.is_some() {
-			return;
-		}
-
-		let sorted_handlers = get_sorted_handlers(&self.input, distance_calculator);
-		self.input.set_handler_order(sorted_handlers.iter());
-	}
-}

src/objects/input/sk_hand.rs

@@ -1,237 +0,0 @@
-use crate::core::client::INTERNAL_CLIENT;
-use crate::nodes::fields::{Field, FieldTrait};
-use crate::nodes::input::{InputDataType, InputHandler, INPUT_HANDLER_REGISTRY};
-use crate::nodes::OwnedNode;
-use crate::nodes::{
-	input::{Hand, InputMethod, Joint},
-	spatial::Spatial,
-	Node,
-};
-use crate::objects::{ObjectHandle, SpatialRef};
-use color_eyre::eyre::Result;
-use glam::{Mat4, Quat, Vec3};
-use serde::{Deserialize, Serialize};
-use stardust_xr::values::Datamap;
-use std::f32::INFINITY;
-use std::sync::Arc;
-use stereokit_rust::sk::{DisplayMode, MainThreadToken, Sk};
-use stereokit_rust::system::{HandJoint, HandSource, Handed, Input, LinePoint, Lines};
-use stereokit_rust::util::Color128;
-use zbus::Connection;
-
-use super::{get_sorted_handlers, CaptureManager};
-
-fn convert_joint(joint: HandJoint) -> Joint {
-	Joint {
-		position: Vec3::from(joint.position).into(),
-		rotation: Quat::from(joint.orientation).into(),
-		radius: joint.radius,
-		distance: 0.0,
-	}
-}
-
-#[derive(Default, Deserialize, Serialize)]
-struct HandDatamap {
-	pinch_strength: f32,
-	grab_strength: f32,
-}
-
-pub struct SkHand {
-	_node: OwnedNode,
-	palm_spatial: Arc<Spatial>,
-	palm_object: ObjectHandle<SpatialRef>,
-	handed: Handed,
-	input: Arc<InputMethod>,
-	capture_manager: CaptureManager,
-	datamap: HandDatamap,
-}
-impl SkHand {
-	pub fn new(connection: &Connection, handed: Handed) -> Result<Self> {
-		let (palm_spatial, palm_object) = SpatialRef::create(
-			connection,
-			&("/org/stardustxr/Hand/".to_string()
-				+ match handed {
-					Handed::Left => "left",
-					_ => "right",
-				} + "/palm"),
-		);
-		let _node = Node::generate(&INTERNAL_CLIENT, false).add_to_scenegraph_owned()?;
-		Spatial::add_to(&_node.0, None, Mat4::IDENTITY, false);
-		let hand = InputDataType::Hand(Hand {
-			right: handed == Handed::Right,
-			..Default::default()
-		});
-		let datamap = Datamap::from_typed(HandDatamap::default())?;
-		let input = InputMethod::add_to(&_node.0, hand, datamap)?;
-
-		Input::hand_visible(handed, false);
-		Ok(SkHand {
-			_node,
-			palm_spatial,
-			palm_object,
-			handed,
-			input,
-			capture_manager: CaptureManager::default(),
-			datamap: Default::default(),
-		})
-	}
-	pub fn update(&mut self, sk: &Sk, token: &MainThreadToken) {
-		let sk_hand = Input::hand(self.handed);
-		let real_hand = Input::hand_source(self.handed) as u32 == HandSource::Articulated as u32;
-		if let InputDataType::Hand(hand) = &mut *self.input.data.lock() {
-			let input_node = self.input.spatial.node().unwrap();
-			input_node.set_enabled(
-				(real_hand || sk.get_active_display_mode() == DisplayMode::Flatscreen)
-					&& sk_hand.tracked.is_active(),
-			);
-			if input_node.enabled() {
-				hand.thumb.tip = convert_joint(sk_hand.fingers[0][4]);
-				hand.thumb.distal = convert_joint(sk_hand.fingers[0][3]);
-				hand.thumb.proximal = convert_joint(sk_hand.fingers[0][2]);
-				hand.thumb.metacarpal = convert_joint(sk_hand.fingers[0][1]);
-
-				for (finger, mut sk_finger) in [
-					(&mut hand.index, sk_hand.fingers[1]),
-					(&mut hand.middle, sk_hand.fingers[2]),
-					(&mut hand.ring, sk_hand.fingers[3]),
-					(&mut hand.little, sk_hand.fingers[4]),
-				] {
-					sk_finger[4].radius = 0.0;
-					finger.tip = convert_joint(sk_finger[4]);
-					finger.distal = convert_joint(sk_finger[3]);
-					finger.intermediate = convert_joint(sk_finger[2]);
-					finger.proximal = convert_joint(sk_finger[1]);
-					finger.metacarpal = convert_joint(sk_finger[0]);
-				}
-
-				hand.palm.position = Vec3::from(sk_hand.palm.position).into();
-				hand.palm.rotation = Quat::from(sk_hand.palm.orientation).into();
-				hand.palm.radius =
-					(sk_hand.fingers[2][0].radius + sk_hand.fingers[2][1].radius) * 0.5;
-
-				self.palm_spatial
-					.set_local_transform(Mat4::from_rotation_translation(
-						hand.palm.rotation.into(),
-						hand.palm.position.into(),
-					));
-
-				hand.wrist.position = Vec3::from(sk_hand.wrist.position).into();
-				hand.wrist.rotation = Quat::from(sk_hand.wrist.orientation).into();
-				hand.wrist.radius =
-					(sk_hand.fingers[0][0].radius + sk_hand.fingers[4][0].radius) * 0.5;
-
-				hand.elbow = None;
-
-				self.draw(
-					token,
-					if self.capture_manager.capture.is_none() {
-						Color128::new_rgb(1.0, 1.0, 1.0)
-					} else {
-						Color128::new_rgb(0.0, 1.0, 0.75)
-					},
-					hand,
-				);
-			}
-		}
-		self.datamap.pinch_strength = sk_hand.pinch_activation;
-		self.datamap.grab_strength = sk_hand.grip_activation;
-		*self.input.datamap.lock() = Datamap::from_typed(&self.datamap).unwrap();
-
-		let distance_calculator = |space: &Arc<Spatial>, data: &InputDataType, field: &Field| {
-			let InputDataType::Hand(hand) = data else {
-				return None;
-			};
-			let thumb_tip_distance = field.distance(space, hand.thumb.tip.position.into());
-			let index_tip_distance = field.distance(space, hand.index.tip.position.into());
-			let middle_tip_distance = field.distance(space, hand.middle.tip.position.into());
-			let ring_tip_distance = field.distance(space, hand.ring.tip.position.into());
-
-			Some(
-				(thumb_tip_distance * 0.3)
-					+ (index_tip_distance * 0.4)
-					+ (middle_tip_distance * 0.15)
-					+ (ring_tip_distance * 0.15),
-			)
-		};
-
-		self.capture_manager.update_capture(&self.input);
-		self.capture_manager
-			.set_new_capture(&self.input, distance_calculator);
-		self.capture_manager.apply_capture(&self.input);
-
-		if self.capture_manager.capture.is_some() {
-			return;
-		}
-
-		let sorted_handlers = get_sorted_handlers(&self.input, distance_calculator);
-		self.input.set_handler_order(sorted_handlers.iter());
-	}
-
-	fn draw(&self, token: &MainThreadToken, color: Color128, hand: &Hand) {
-		// thumb
-		Lines::add_list(
-			token,
-			&[
-				joint_to_line_point(&hand.thumb.tip, color),
-				joint_to_line_point(&hand.thumb.distal, color),
-				joint_to_line_point(&hand.thumb.proximal, color),
-				joint_to_line_point(&hand.thumb.metacarpal, color),
-			],
-		);
-		// index
-		Lines::add_list(
-			token,
-			&[
-				joint_to_line_point(&hand.index.tip, color),
-				joint_to_line_point(&hand.index.distal, color),
-				joint_to_line_point(&hand.index.intermediate, color),
-				joint_to_line_point(&hand.index.proximal, color),
-				joint_to_line_point(&hand.index.metacarpal, color),
-			],
-		);
-		// middle
-		Lines::add_list(
-			token,
-			&[
-				joint_to_line_point(&hand.middle.tip, color),
-				joint_to_line_point(&hand.middle.distal, color),
-				joint_to_line_point(&hand.middle.intermediate, color),
-				joint_to_line_point(&hand.middle.proximal, color),
-				joint_to_line_point(&hand.middle.metacarpal, color),
-			],
-		);
-		// ring
-		Lines::add_list(
-			token,
-			&[
-				joint_to_line_point(&hand.ring.tip, color),
-				joint_to_line_point(&hand.ring.distal, color),
-				joint_to_line_point(&hand.ring.intermediate, color),
-				joint_to_line_point(&hand.ring.proximal, color),
-				joint_to_line_point(&hand.ring.metacarpal, color),
-			],
-		);
-
-		// palm
-		Lines::add_list(
-			token,
-			&[
-				joint_to_line_point(&hand.wrist, color),
-				joint_to_line_point(&hand.thumb.metacarpal, color),
-				joint_to_line_point(&hand.index.metacarpal, color),
-				joint_to_line_point(&hand.middle.metacarpal, color),
-				joint_to_line_point(&hand.ring.metacarpal, color),
-				joint_to_line_point(&hand.little.metacarpal, color),
-				joint_to_line_point(&hand.wrist, color),
-			],
-		);
-	}
-}
-
-fn joint_to_line_point(joint: &Joint, color: Color128) -> LinePoint {
-	LinePoint {
-		pt: Vec3::from(joint.position).into(),
-		thickness: joint.radius * 2.0,
-		color: color.into(),
-	}
-}

src/objects/mod.rs

@@ -3,187 +3,36 @@
 use crate::{
 	core::client::INTERNAL_CLIENT,
 	nodes::{
-		fields::{Field, Shape, EXPORTED_FIELDS},
-		spatial::{Spatial, EXPORTED_SPATIALS},
 		Node, OwnedNode,
+		fields::{EXPORTED_FIELDS, Field, Shape},
+		spatial::{EXPORTED_SPATIALS, Spatial},
 	},
 };
-use glam::{vec3, Mat4};
+use glam::{Mat4, vec3};
 use input::{
-	eye_pointer::EyePointer, mouse_pointer::MousePointer, sk_controller::SkController,
-	sk_hand::SkHand,
+	eye_pointer::EyePointer, mouse_pointer::MousePointer, oxr_controller::OxrControllerInput,
+	oxr_hand::OxrHandInput,
 };
+use parking_lot::RwLock;
 use play_space::PlaySpaceBounds;
-use std::{marker::PhantomData, sync::Arc};
-use stereokit_rust::{
-	sk::{DisplayMode, MainThreadToken, Sk},
-	system::{Handed, Input, Key, World},
-	util::Device,
+use stardust_xr::schemas::dbus::object_registry::ObjectRegistry;
+use std::{
+	marker::PhantomData,
+	sync::{Arc, atomic::Ordering},
 };
-use zbus::{interface, object_server::Interface, zvariant::OwnedObjectPath, Connection};
+use zbus::{Connection, interface, object_server::Interface, zvariant::OwnedObjectPath};
 
+pub mod hmd;
 pub mod input;
 pub mod play_space;
 
-enum Inputs {
-	XR {
-		controller_left: SkController,
-		controller_right: SkController,
-		hand_left: SkHand,
-		hand_right: SkHand,
-		eye_pointer: Option<EyePointer>,
-	},
-	MousePointer(MousePointer),
-	// Controllers((SkController, SkController)),
-	Hands {
-		left: SkHand,
-		right: SkHand,
-	},
-}
-
-pub struct ServerObjects {
-	connection: Connection,
-	hmd: (Arc<Spatial>, ObjectHandle<SpatialRef>),
-	play_space: Option<(Arc<Spatial>, ObjectHandle<SpatialRef>)>,
-	inputs: Inputs,
-	disable_controllers: bool,
-	disable_hands: bool,
-}
-impl ServerObjects {
-	pub fn new(
-		connection: Connection,
-		sk: &Sk,
-		disable_controllers: bool,
-		disable_hands: bool,
-	) -> ServerObjects {
-		let hmd = SpatialRef::create(&connection, "/org/stardustxr/HMD");
-
-		let play_space = (World::has_bounds()
-			&& World::get_bounds_size().x != 0.0
-			&& World::get_bounds_size().y != 0.0)
-			.then(|| SpatialRef::create(&connection, "/org/stardustxr/PlaySpace"));
-		if play_space.is_some() {
-			let dbus_connection = connection.clone();
-			tokio::task::spawn(async move {
-				PlaySpaceBounds::create(&dbus_connection).await;
-				dbus_connection
-					.request_name("org.stardustxr.PlaySpace")
-					.await
-					.unwrap();
-			});
-		}
-
-		tokio::task::spawn({
-			let connection = connection.clone();
-			async move {
-				connection
-					.request_name("org.stardustxr.Controllers")
-					.await
-					.unwrap();
-				connection
-					.request_name("org.stardustxr.Hands")
-					.await
-					.unwrap();
-			}
-		});
-
-		let inputs = if sk.get_active_display_mode() == DisplayMode::MixedReality {
-			Inputs::XR {
-				controller_left: SkController::new(&connection, Handed::Left).unwrap(),
-				controller_right: SkController::new(&connection, Handed::Right).unwrap(),
-				hand_left: SkHand::new(&connection, Handed::Left).unwrap(),
-				hand_right: SkHand::new(&connection, Handed::Right).unwrap(),
-				eye_pointer: Device::has_eye_gaze()
-					.then(EyePointer::new)
-					.transpose()
-					.unwrap(),
-			}
-		} else {
-			Inputs::MousePointer(MousePointer::new().unwrap())
-		};
-
-		ServerObjects {
-			connection,
-			hmd,
-			play_space,
-			inputs,
-			disable_controllers,
-			disable_hands,
-		}
-	}
-
-	pub fn update(&mut self, sk: &Sk, token: &MainThreadToken) {
-		let hmd_pose = Input::get_head();
-		self.hmd
-			.0
-			.set_local_transform(Mat4::from_scale_rotation_translation(
-				vec3(1.0, 1.0, 1.0),
-				hmd_pose.orientation.into(),
-				hmd_pose.position.into(),
-			));
-
-		if let Some(play_space) = self.play_space.as_ref() {
-			let pose = World::get_bounds_pose();
-			play_space
-				.0
-				.set_local_transform(Mat4::from_rotation_translation(
-					pose.orientation.into(),
-					pose.position.into(),
-				));
-		}
-
-		if sk.get_active_display_mode() != DisplayMode::MixedReality {
-			if Input::key(Key::F6).is_just_inactive() {
-				self.inputs = Inputs::MousePointer(MousePointer::new().unwrap());
-			}
-			// if Input::key(Key::F7).is_just_inactive() {
-			// 	self.inputs = Inputs::Controllers((
-			// 		SkController::new(Handed::Left).unwrap(),
-			// 		SkController::new(Handed::Right).unwrap(),
-			// 	));
-			// }
-			if Input::key(Key::F8).is_just_inactive() {
-				self.inputs = Inputs::Hands {
-					left: SkHand::new(&self.connection, Handed::Left).unwrap(),
-					right: SkHand::new(&self.connection, Handed::Right).unwrap(),
-				};
-			}
-		}
-
-		match &mut self.inputs {
-			Inputs::XR {
-				controller_left,
-				controller_right,
-				hand_left,
-				hand_right,
-				eye_pointer,
-			} => {
-				if !self.disable_controllers {
-					controller_left.update(token);
-					controller_right.update(token);
-				}
-				if !self.disable_hands {
-					hand_left.update(sk, token);
-					hand_right.update(sk, token);
-				}
-				if let Some(eye_pointer) = eye_pointer {
-					eye_pointer.update();
-				}
-			}
-			Inputs::MousePointer(mouse_pointer) => mouse_pointer.update(),
-			// Inputs::Controllers((left, right)) => {
-			// 	left.update(token);
-			// 	right.update(token);
-			// }
-			Inputs::Hands { left, right } => {
-				left.update(sk, token);
-				right.update(sk, token);
-			}
-		}
-	}
-}
-
 pub struct ObjectHandle<I: Interface>(Connection, OwnedObjectPath, PhantomData<I>);
+
+impl<I: Interface> Clone for ObjectHandle<I> {
+	fn clone(&self) -> Self {
+		Self(self.0.clone(), self.1.clone(), PhantomData)
+	}
+}
 impl<I: Interface> Drop for ObjectHandle<I> {
 	fn drop(&mut self) {
 		let connection = self.0.clone();
@@ -200,7 +49,9 @@ impl SpatialRef {
 		let node = OwnedNode(Arc::new(Node::generate(&INTERNAL_CLIENT, false)));
 		let spatial = Spatial::add_to(&node.0, None, Mat4::IDENTITY, false);
 		let uid: u64 = rand::random();
-		EXPORTED_SPATIALS.lock().insert(uid, node.0.clone());
+		EXPORTED_SPATIALS
+			.lock()
+			.insert(uid, Arc::downgrade(&node.0));
 
 		tokio::task::spawn({
 			let connection = connection.clone();
@@ -231,6 +82,52 @@ impl SpatialRef {
 	}
 }
 
+pub struct Tracked(bool);
+impl Tracked {
+	pub fn new(connection: &Connection, path: &str) -> ObjectHandle<Tracked> {
+		tokio::task::spawn({
+			let connection = connection.clone();
+			let path = path.to_string();
+			async move {
+				connection
+					.object_server()
+					.at(path, Self(false))
+					.await
+					.unwrap();
+			}
+		});
+		ObjectHandle(
+			connection.clone(),
+			OwnedObjectPath::try_from(path.to_string()).unwrap(),
+			PhantomData,
+		)
+	}
+}
+impl ObjectHandle<Tracked> {
+	pub async fn set_tracked(&self, is_tracked: bool) -> zbus::Result<()> {
+		let tracked_ref = self
+			.0
+			.object_server()
+			.interface::<_, Tracked>(self.1.as_ref())
+			.await?;
+		let mut tracked = tracked_ref.get_mut().await;
+		if tracked.0 != is_tracked {
+			tracked.0 = is_tracked;
+			tracked
+				.is_tracked_changed(tracked_ref.signal_emitter())
+				.await;
+		}
+		Ok(())
+	}
+}
+#[interface(name = "org.stardustxr.Tracked")]
+impl Tracked {
+	#[zbus(property)]
+	fn is_tracked(&self) -> bool {
+		self.0
+	}
+}
+
 pub struct FieldRef(u64, OwnedNode);
 impl FieldRef {
 	pub fn create(
@@ -242,7 +139,7 @@ impl FieldRef {
 		Spatial::add_to(&node.0, None, Mat4::IDENTITY, false);
 		let field = Field::add_to(&node.0, shape).unwrap();
 		let uid: u64 = rand::random();
-		EXPORTED_FIELDS.lock().insert(uid, node.0.clone());
+		EXPORTED_FIELDS.lock().insert(uid, Arc::downgrade(&node.0));
 
 		tokio::task::spawn({
 			let connection = connection.clone();

src/objects/play_space.rs

@@ -1,12 +1,154 @@
-use stereokit_rust::system::World;
-use zbus::{interface, Connection, ObjectServer};
+use std::sync::Arc;
 
-pub struct PlaySpaceBounds;
+use bevy::prelude::*;
+use bevy_mod_openxr::{
+	helper_traits::{ToQuat, ToVec3},
+	resources::OxrFrameState,
+	session::OxrSession,
+};
+use bevy_mod_xr::{
+	session::{XrPreDestroySession, XrSessionCreated},
+	spaces::{XrPrimaryReferenceSpace, XrReferenceSpace, XrSpace},
+};
+use openxr::SpaceLocationFlags;
+use parking_lot::RwLock;
+use zbus::{Connection, ObjectServer, interface};
+
+use crate::{DbusConnection, PreFrameWait, nodes::spatial::Spatial};
+
+use super::{ObjectHandle, SpatialRef, Tracked};
+
+pub struct PlaySpacePlugin;
+impl Plugin for PlaySpacePlugin {
+	fn build(&self, app: &mut App) {
+		app.add_systems(XrPreDestroySession, destroy_stage_space);
+		app.add_systems(XrSessionCreated, create_stage_space);
+		app.add_systems(PreFrameWait, update);
+		app.add_systems(Startup, setup);
+	}
+}
+
+fn setup(connection: Res<DbusConnection>, mut cmds: Commands) {
+	let (spatial, spatial_handle) = SpatialRef::create(&connection, "/org/stardustxr/PlaySpace");
+	// the OpenXR session might not exist quite yet
+	let tracked = Tracked::new(&connection, "/org/stardustxr/PlaySpace");
+	let dbus_connection = connection.clone();
+	let play_space_data = Arc::new(RwLock::default());
+	tokio::task::spawn({
+		let data = play_space_data.clone();
+		async move {
+			PlaySpaceBounds::create(&dbus_connection, data).await;
+			dbus_connection
+				.request_name("org.stardustxr.PlaySpace")
+				.await
+				.unwrap();
+		}
+	});
+	cmds.insert_resource(PlaySpace {
+		spatial,
+		_spatial_handle: spatial_handle,
+		tracked_handle: tracked,
+		bounds: play_space_data,
+	});
+}
+
+#[derive(Resource)]
+struct StageSpace(XrSpace);
+fn create_stage_space(session: Res<OxrSession>, mut cmds: Commands) {
+	let space = session
+		.create_reference_space(openxr::ReferenceSpaceType::STAGE, Transform::IDENTITY)
+		.inspect_err(|err| error!("failed to create Stage XrSpace"))
+		.ok();
+	if let Some(space) = space {
+		cmds.insert_resource(StageSpace(space.0));
+	}
+}
+fn destroy_stage_space(session: Res<OxrSession>, mut cmds: Commands, stage: Res<StageSpace>) {
+	session.destroy_space(stage.0);
+	cmds.remove_resource::<StageSpace>();
+}
+
+/// TODO: impl this
+fn update(
+	session: Option<Res<OxrSession>>,
+	stage: Option<Res<StageSpace>>,
+	ref_space: Option<Res<XrPrimaryReferenceSpace>>,
+	play_space: Res<PlaySpace>,
+	state: Option<Res<OxrFrameState>>,
+) {
+	let (Some(session), Some(stage), Some(ref_space), Some(state)) =
+		(session, stage, ref_space, state)
+	else {
+		play_space.bounds.write().drain(..);
+		tokio::task::spawn({
+			let handle = play_space.tracked_handle.clone();
+			async move {
+				handle.set_tracked(false).await;
+			}
+		});
+
+		play_space
+			.spatial
+			.set_local_transform(Mat4::from_translation(vec3(0.0, -1.65, 0.0)));
+		return;
+	};
+	// this won't be correct with pipelined rendering
+	let location = session
+		.locate_space(&stage.0, &ref_space, state.predicted_display_time)
+		.inspect_err(|err| error!("Error while Locating OpenXR Stage Space {err}"));
+	if let Ok(location) = location {
+		let is_tracked = location.location_flags.contains(
+			SpaceLocationFlags::POSITION_VALID
+				| SpaceLocationFlags::POSITION_TRACKED
+				| SpaceLocationFlags::ORIENTATION_VALID
+				| SpaceLocationFlags::ORIENTATION_TRACKED,
+		);
+		tokio::task::spawn({
+			let handle = play_space.tracked_handle.clone();
+			async move {
+				handle.set_tracked(is_tracked).await;
+			}
+		});
+		if is_tracked {
+			play_space
+				.spatial
+				.set_local_transform(Mat4::from_rotation_translation(
+					location.pose.orientation.to_quat(),
+					location.pose.position.to_vec3(),
+				));
+		}
+	}
+	// session.reference_space_bounds_rect(openxr::ReferenceSpaceType::STAGE);
+
+	// if (World::has_bounds()
+	// 	&& World::get_bounds_size().x != 0.0
+	// 	&& World::get_bounds_size().y != 0.0)
+	// {
+	// 	let bounds = World::get_bounds_size();
+	// 	vec![
+	// 		((bounds.x).into(), (bounds.y).into()),
+	// 		((bounds.x).into(), (-bounds.y).into()),
+	// 		((-bounds.x).into(), (-bounds.y).into()),
+	// 		((-bounds.x).into(), (bounds.y).into()),
+	// 	]
+	// } else {
+	// 	vec![]
+	// }
+}
+
+#[derive(Resource)]
+pub struct PlaySpace {
+	spatial: Arc<Spatial>,
+	_spatial_handle: ObjectHandle<SpatialRef>,
+	tracked_handle: ObjectHandle<Tracked>,
+	bounds: Arc<RwLock<Vec<(f64, f64)>>>,
+}
+pub struct PlaySpaceBounds(Arc<RwLock<Vec<(f64, f64)>>>);
 impl PlaySpaceBounds {
-	pub async fn create(connection: &Connection) {
+	pub async fn create(connection: &Connection, data: Arc<RwLock<Vec<(f64, f64)>>>) {
 		connection
 			.object_server()
-			.at("/org/stardustxr/PlaySpace", Self)
+			.at("/org/stardustxr/PlaySpace", Self(data))
 			.await
 			.unwrap();
 	}
@@ -15,12 +157,6 @@ impl PlaySpaceBounds {
 impl PlaySpaceBounds {
 	#[zbus(property)]
 	fn bounds(&self) -> Vec<(f64, f64)> {
-		let bounds = World::get_bounds_size();
-		vec![
-			((bounds.x).into(), (bounds.y).into()),
-			((bounds.x).into(), (-bounds.y).into()),
-			((-bounds.x).into(), (-bounds.y).into()),
-			((-bounds.x).into(), (bounds.y).into()),
-		]
+		self.0.read().clone()
 	}
 }

src/session.rs

@@ -86,14 +86,11 @@ pub fn run_client(mut command: Command, debug_launched_clients: bool) -> Option<
 }
 
 pub fn connection_env() -> FxHashMap<String, String> {
-	macro_rules! var_env_insert {
-		($env:ident, $name:ident) => {
-			$env.insert(stringify!($name).to_string(), $name.get().unwrap().clone());
-		};
-	}
-
 	let mut env: FxHashMap<String, String> = FxHashMap::default();
-	var_env_insert!(env, STARDUST_INSTANCE);
+	env.insert(
+		stringify!(STARDUST_INSTANCE).to_string(),
+		STARDUST_INSTANCE.get().unwrap().clone(),
+	);
 
 	if let Some(flat_wayland_display) = std::env::var_os("WAYLAND_DISPLAY") {
 		env.insert(
@@ -103,12 +100,11 @@ pub fn connection_env() -> FxHashMap<String, String> {
 	}
 	#[cfg(feature = "wayland")]
 	{
-		var_env_insert!(env, WAYLAND_DISPLAY);
-		env.insert("GDK_BACKEND".to_string(), "wayland".to_string());
-		env.insert("QT_QPA_PLATFORM".to_string(), "wayland".to_string());
-		env.insert("MOZ_ENABLE_WAYLAND".to_string(), "1".to_string());
-		env.insert("CLUTTER_BACKEND".to_string(), "wayland".to_string());
-		env.insert("SDL_VIDEODRIVER".to_string(), "wayland".to_string());
+		env.insert(
+			stringify!(WAYLAND_DISPLAY).to_string(),
+			WAYLAND_DISPLAY.get().unwrap().to_string_lossy().to_string(),
+		);
+		env.insert("XDG_SESSION_TYPE".to_string(), "wayland".to_string());
 	}
 	env
 }

src/tracking_offset.rs

@@ -0,0 +1,82 @@
+use bevy::{
+	app::{Plugin, Update},
+	ecs::{
+		resource::Resource,
+		schedule::{Condition, IntoScheduleConfigs, common_conditions::resource_exists},
+		system::{Commands, Res, ResMut},
+	},
+	transform::components::Transform,
+};
+use bevy_mod_openxr::{
+	helper_traits::ToTransform as _,
+	poll_events::{OxrEventHandlerExt, OxrEventIn},
+	resources::OxrFrameState,
+	session::OxrSession,
+};
+use bevy_mod_xr::{session::XrSessionCreated, spaces::XrPrimaryReferenceSpace};
+use glam::{Quat, Vec3};
+use openxr::ReferenceSpaceType;
+
+pub struct TrackingOffsetPlugin;
+
+impl Plugin for TrackingOffsetPlugin {
+	fn build(&self, app: &mut bevy::app::App) {
+		app.add_oxr_event_handler(reset_offset);
+		app.add_systems(XrSessionCreated, |mut cmds: Commands| {
+			cmds.insert_resource(OffsetTag);
+		});
+		app.add_systems(
+			Update,
+			offset.run_if(resource_exists::<OffsetTag>.and(resource_exists::<OxrFrameState>)),
+		);
+	}
+}
+
+#[derive(Resource)]
+struct OffsetTag;
+
+fn reset_offset(
+	oxr_event: OxrEventIn,
+	mut ref_space: ResMut<XrPrimaryReferenceSpace>,
+	session: Res<OxrSession>,
+) {
+	if let openxr::Event::ReferenceSpaceChangePending(v) = *oxr_event
+		&& v.reference_space_type() == ReferenceSpaceType::LOCAL
+	{
+		let space = session
+			.create_reference_space(ReferenceSpaceType::LOCAL, Transform::IDENTITY)
+			.unwrap();
+		session.destroy_space(ref_space.0.0).unwrap();
+		ref_space.0 = space;
+	}
+}
+
+fn offset(
+	session: Res<OxrSession>,
+	state: Res<OxrFrameState>,
+	mut primary_ref_space: ResMut<XrPrimaryReferenceSpace>,
+	mut cmds: Commands,
+) {
+	cmds.remove_resource::<OffsetTag>();
+	let local = session
+		.create_reference_space(ReferenceSpaceType::LOCAL, Transform::IDENTITY)
+		.unwrap();
+	let view = session
+		.create_reference_space(ReferenceSpaceType::VIEW, Transform::IDENTITY)
+		.unwrap();
+	let view_pose = session
+		.locate_space(&view, &local, state.predicted_display_time)
+		.unwrap()
+		.pose
+		.to_transform();
+	let offset = view_pose.with_rotation(Quat::from_axis_angle(
+		Vec3::Y,
+		view_pose.rotation.to_euler(glam::EulerRot::XYZ).1,
+	));
+	let offset = Transform::from_matrix(offset.compute_matrix());
+	let local_offset = session
+		.create_reference_space(ReferenceSpaceType::LOCAL, offset)
+		.unwrap();
+	session.destroy_space(primary_ref_space.0.0).unwrap();
+	primary_ref_space.0 = local_offset;
+}

src/wayland/compositor.rs

@@ -1,127 +0,0 @@
-use super::{
-	state::{ClientState, WaylandState},
-	utils::{ChildInfoExt, WlSurfaceExt},
-	xdg_shell::surface_panel_item,
-};
-use crate::{
-	nodes::items::panel::{ChildInfo, Geometry, SurfaceId},
-	wayland::surface::CoreSurface,
-};
-use parking_lot::Mutex;
-use portable_atomic::{AtomicU32, Ordering};
-use rand::Rng;
-use smithay::{
-	backend::renderer::utils::{on_commit_buffer_handler, RendererSurfaceStateUserData},
-	delegate_compositor,
-	reexports::wayland_server::{protocol::wl_surface::WlSurface, Client},
-	wayland::compositor::{
-		self, add_post_commit_hook, CompositorClientState, CompositorHandler, CompositorState,
-	},
-};
-use std::sync::Arc;
-use tracing::debug;
-
-impl CompositorHandler for WaylandState {
-	fn compositor_state(&mut self) -> &mut CompositorState {
-		&mut self.compositor_state
-	}
-
-	fn commit(&mut self, surface: &WlSurface) {
-		debug!(?surface, "Surface commit");
-
-		on_commit_buffer_handler::<WaylandState>(surface);
-		let mut count = 0;
-		compositor::with_states(surface, |data| {
-			let count_new = data
-				.data_map
-				.insert_if_missing_threadsafe(|| AtomicU32::new(0));
-			if !count_new {
-				if let Some(stored_count) = data.data_map.get::<AtomicU32>() {
-					count = stored_count.fetch_add(1, Ordering::Relaxed);
-				}
-			}
-
-			data.data_map.get::<Arc<CoreSurface>>().cloned()
-		});
-	}
-
-	fn client_compositor_state<'a>(&self, client: &'a Client) -> &'a CompositorClientState {
-		&client.get_data::<ClientState>().unwrap().compositor_state
-	}
-
-	fn new_subsurface(&mut self, surface: &WlSurface, parent: &WlSurface) {
-		let id = rand::thread_rng().gen_range(0..u64::MAX);
-		surface.insert_data(SurfaceId::Child(id));
-		CoreSurface::add_to(surface);
-		let Some(parent_surface_id) = parent.get_data::<SurfaceId>() else {
-			return;
-		};
-		surface.insert_data(Mutex::new(ChildInfo {
-			id,
-			parent: parent_surface_id,
-			geometry: Geometry {
-				origin: [0; 2].into(),
-				size: [256; 2].into(),
-			},
-			z_order: 1,
-			receives_input: false,
-		}));
-
-		let Some(panel_item) = surface_panel_item(parent) else {
-			return;
-		};
-		let panel_item_weak = Arc::downgrade(&panel_item);
-		add_post_commit_hook(surface, move |_: &mut WaylandState, _dh, surf| {
-			if surface_panel_item(surf).is_some() {
-				return;
-			}
-			surf.insert_data(panel_item_weak.clone());
-
-			let Some(panel_item) = surface_panel_item(surf) else {
-				return;
-			};
-			panel_item.backend.new_child(surf);
-		});
-
-		add_post_commit_hook(surface, move |_: &mut WaylandState, _dh, surf| {
-			let Some(view) = surf
-				.get_data_raw::<RendererSurfaceStateUserData, _, _>(|s| s.lock().ok()?.view())
-				.flatten()
-			else {
-				return;
-			};
-			let mut changed = false;
-			surf.with_child_info(|info| {
-				if info.geometry.origin.x != view.offset.x
-					&& info.geometry.origin.y != view.offset.y
-				{
-					changed = true;
-				}
-				if info.geometry.size.x != view.dst.w as u32
-					&& info.geometry.size.y != view.dst.h as u32
-				{
-					changed = true;
-				}
-				info.geometry.size = [view.dst.w as u32, view.dst.h as u32].into();
-			});
-
-			let Some(panel_item) = surface_panel_item(surf) else {
-				return;
-			};
-			if changed {
-				panel_item.backend.reposition_child(surf);
-			}
-		});
-	}
-
-	fn destroyed(&mut self, surface: &WlSurface) {
-		let Some(panel_item) = surface_panel_item(surface) else {
-			return;
-		};
-		if surface.get_child_info().is_some() {
-			panel_item.backend.drop_child(surface);
-		}
-	}
-}
-
-delegate_compositor!(WaylandState);

src/wayland/core/buffer.rs

@@ -0,0 +1,94 @@
+use crate::wayland::Message;
+use crate::wayland::dmabuf::buffer_backing::DmabufBacking;
+use crate::wayland::{MessageSink, core::shm_buffer_backing::ShmBufferBacking, util::ClientExt};
+use bevy::{
+	asset::{Assets, Handle},
+	image::Image,
+};
+use bevy_dmabuf::import::ImportedDmatexs;
+use mint::Vector2;
+use std::sync::Arc;
+pub use waynest::server::protocol::core::wayland::wl_buffer::*;
+use waynest::{
+	server::{Client, Dispatcher, Result},
+	wire::ObjectId,
+};
+
+#[derive(Debug)]
+pub struct BufferUsage {
+	pub buffer: Arc<Buffer>,
+	message_sink: MessageSink,
+}
+impl BufferUsage {
+	pub fn new(client: &Client, buffer: &Arc<Buffer>) -> Arc<Self> {
+		Arc::new(Self {
+			buffer: buffer.clone(),
+			message_sink: client.message_sink(),
+		})
+	}
+}
+impl Drop for BufferUsage {
+	fn drop(&mut self) {
+		let _ = self
+			.message_sink
+			.send(Message::ReleaseBuffer(self.buffer.clone()));
+	}
+}
+
+#[derive(Debug)]
+pub enum BufferBacking {
+	Shm(ShmBufferBacking),
+	Dmabuf(DmabufBacking),
+}
+
+#[derive(Debug, Dispatcher)]
+pub struct Buffer {
+	pub id: ObjectId,
+	backing: BufferBacking,
+}
+
+impl Buffer {
+	#[tracing::instrument(level = "debug", skip_all)]
+	pub fn new(client: &mut Client, id: ObjectId, backing: BufferBacking) -> Arc<Self> {
+		client.insert(id, Self { id, backing })
+	}
+
+	/// Returns the tex if it was updated
+	#[tracing::instrument(level = "debug", skip_all)]
+	pub fn update_tex(
+		&self,
+		dmatexes: &ImportedDmatexs,
+		images: &mut Assets<Image>,
+	) -> Option<Handle<Image>> {
+		tracing::debug!("Updating texture for buffer {:?}", self.id);
+		match &self.backing {
+			BufferBacking::Shm(backing) => backing.update_tex(images),
+			BufferBacking::Dmabuf(backing) => backing.update_tex(dmatexes, images),
+		}
+	}
+
+	pub fn is_transparent(&self) -> bool {
+		match &self.backing {
+			BufferBacking::Shm(backing) => backing.is_transparent(),
+			BufferBacking::Dmabuf(backing) => backing.is_transparent(),
+		}
+	}
+
+	pub fn size(&self) -> Vector2<usize> {
+		match &self.backing {
+			BufferBacking::Shm(backing) => backing.size(),
+			BufferBacking::Dmabuf(backing) => backing.size(),
+		}
+	}
+	pub fn uses_buffer_usage(&self) -> bool {
+		matches!(self.backing, BufferBacking::Dmabuf(_))
+	}
+}
+
+impl WlBuffer for Buffer {
+	/// https://wayland.app/protocols/wayland#wl_buffer:request:destroy
+	async fn destroy(&self, _client: &mut Client, _sender_id: ObjectId) -> Result<()> {
+		tracing::info!("Destroying buffer {:?}", self.id);
+		Ok(())
+	}
+}

src/wayland/core/callback.rs

@@ -0,0 +1,10 @@
+pub use waynest::server::protocol::core::wayland::wl_callback::*;
+use waynest::{
+	server::{Dispatcher, Result},
+	wire::ObjectId,
+};
+
+#[derive(Debug, Dispatcher, Clone)]
+pub struct Callback(pub ObjectId);
+/// https://wayland.app/protocols/wayland#wl_callback
+impl WlCallback for Callback {}

src/wayland/core/compositor.rs

@@ -0,0 +1,76 @@
+use super::surface::WL_SURFACE_REGISTRY;
+use crate::wayland::{core::surface::Surface, util::ClientExt};
+pub use waynest::server::protocol::core::wayland::wl_compositor::*;
+use waynest::{
+	server::{
+		Client, Dispatcher, Result,
+		protocol::core::wayland::{wl_region::WlRegion, wl_surface::WlSurface},
+	},
+	wire::ObjectId,
+};
+
+#[derive(Debug, Dispatcher, Default)]
+pub struct Compositor;
+impl WlCompositor for Compositor {
+	/// https://wayland.app/protocols/wayland#wl_compositor:request:create_surface
+	async fn create_surface(
+		&self,
+		client: &mut Client,
+		_sender_id: ObjectId,
+		id: ObjectId,
+	) -> Result<()> {
+		let surface = client.insert(id, Surface::new(client, id));
+		if let Some(output) = client.display().output.get() {
+			surface.enter(client, id, output.id).await?;
+		}
+		WL_SURFACE_REGISTRY.add_raw(&surface);
+
+		Ok(())
+	}
+
+	/// https://wayland.app/protocols/wayland#wl_compositor:request:create_region
+	async fn create_region(
+		&self,
+		client: &mut Client,
+		_sender_id: ObjectId,
+		id: ObjectId,
+	) -> Result<()> {
+		client.insert(id, Region::default());
+		Ok(())
+	}
+}
+
+#[derive(Debug, Dispatcher, Default)]
+pub struct Region {}
+impl WlRegion for Region {
+	/// https://wayland.app/protocols/wayland#wl_region:request:add
+	async fn add(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_x: i32,
+		_y: i32,
+		_width: i32,
+		_height: i32,
+	) -> Result<()> {
+		Ok(())
+	}
+
+	/// https://wayland.app/protocols/wayland#wl_region:request:subtract
+	async fn subtract(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_x: i32,
+		_y: i32,
+		_width: i32,
+		_height: i32,
+	) -> Result<()> {
+		Ok(())
+	}
+
+	/// https://wayland.app/protocols/wayland#wl_region:request:destroy
+	async fn destroy(&self, _client: &mut Client, _sender_id: ObjectId) -> Result<()> {
+		Ok(())
+	}
+}

src/wayland/core/data_device.rs

@@ -0,0 +1,146 @@
+use std::os::fd::OwnedFd;
+use waynest::{
+	server::{
+		Client, Dispatcher, Result,
+		protocol::core::wayland::{
+			wl_data_device::*, wl_data_device_manager::*, wl_data_offer::WlDataOffer,
+			wl_data_source::*,
+		},
+	},
+	wire::ObjectId,
+};
+
+// TODO: actually implement this
+
+#[derive(Debug, Dispatcher)]
+pub struct DataDeviceManager;
+impl WlDataDeviceManager for DataDeviceManager {
+	async fn create_data_source(
+		&self,
+		client: &mut Client,
+		_sender_id: ObjectId,
+		id: ObjectId,
+	) -> Result<()> {
+		client.insert(id, DataSource);
+		Ok(())
+	}
+
+	async fn get_data_device(
+		&self,
+		client: &mut Client,
+		_sender_id: ObjectId,
+		id: ObjectId,
+		_seat: ObjectId,
+	) -> Result<()> {
+		client.insert(id, DataDevice);
+		Ok(())
+	}
+}
+
+#[derive(Debug, Dispatcher)]
+pub struct DataSource;
+impl WlDataSource for DataSource {
+	async fn send(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_mime_type: String,
+		_fd: OwnedFd,
+	) -> Result<()> {
+		Ok(())
+	}
+
+	async fn offer(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_mime_type: String,
+	) -> Result<()> {
+		Ok(())
+	}
+
+	async fn destroy(&self, _client: &mut Client, _sender_id: ObjectId) -> Result<()> {
+		Ok(())
+	}
+
+	async fn set_actions(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_dnd_actions: DndAction,
+	) -> Result<()> {
+		Ok(())
+	}
+}
+
+#[derive(Debug, Dispatcher)]
+pub struct DataDevice;
+impl WlDataDevice for DataDevice {
+	async fn start_drag(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_source: Option<ObjectId>,
+		_origin: ObjectId,
+		_icon: Option<ObjectId>,
+		_serial: u32,
+	) -> Result<()> {
+		Ok(())
+	}
+
+	async fn set_selection(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_source: Option<ObjectId>,
+		_serial: u32,
+	) -> Result<()> {
+		Ok(())
+	}
+
+	async fn release(&self, _client: &mut Client, _sender_id: ObjectId) -> Result<()> {
+		Ok(())
+	}
+}
+
+#[derive(Debug, Dispatcher)]
+pub struct DataOffer;
+impl WlDataOffer for DataOffer {
+	async fn accept(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_serial: u32,
+		_mime_type: Option<String>,
+	) -> Result<()> {
+		Ok(())
+	}
+
+	async fn receive(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_mime_type: String,
+		_fd: OwnedFd,
+	) -> Result<()> {
+		Ok(())
+	}
+
+	async fn destroy(&self, _client: &mut Client, _sender_id: ObjectId) -> Result<()> {
+		Ok(())
+	}
+
+	async fn finish(&self, _client: &mut Client, _sender_id: ObjectId) -> Result<()> {
+		Ok(())
+	}
+
+	async fn set_actions(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_dnd_actions: DndAction,
+		_preferred_action: DndAction,
+	) -> Result<()> {
+		Ok(())
+	}
+}

src/wayland/core/keyboard.rs

@@ -0,0 +1,254 @@
+use crate::{
+	nodes::items::panel::KEYMAPS,
+	wayland::{core::surface::Surface, util::ClientExt},
+};
+use dashmap::{DashMap, DashSet};
+use memfd::MemfdOptions;
+use slotmap::{DefaultKey, KeyData};
+use std::{
+	collections::HashSet,
+	io::Write,
+	os::{
+		fd::IntoRawFd,
+		unix::io::{FromRawFd, OwnedFd},
+	},
+	sync::{Arc, Weak},
+};
+use tokio::sync::Mutex;
+pub use waynest::server::protocol::core::wayland::wl_keyboard::*;
+use waynest::{
+	server::{Client, Dispatcher, Result},
+	wire::ObjectId,
+};
+
+#[derive(Default)]
+struct ModifierState {
+	pressed_keys: HashSet<u32>,
+	mods_depressed: u32,
+	mods_latched: u32,
+	mods_locked: u32,
+	group: u32,
+}
+
+impl ModifierState {
+	fn update_key(&mut self, key: u32, pressed: bool) -> bool {
+		let changed = if pressed {
+			self.pressed_keys.insert(key)
+		} else {
+			self.pressed_keys.remove(&key)
+		};
+
+		if changed {
+			self.update_modifiers();
+		}
+		changed
+	}
+
+	fn update_modifiers(&mut self) {
+		let mut mods = 0;
+
+		// Update modifier state based on currently pressed keys
+		for key in &self.pressed_keys {
+			match *key {
+				input_event_codes::KEY_LEFTSHIFT!() | input_event_codes::KEY_RIGHTSHIFT!() => {
+					mods |= 1
+				}
+				input_event_codes::KEY_LEFTCTRL!() | input_event_codes::KEY_RIGHTCTRL!() => {
+					mods |= 4
+				}
+				input_event_codes::KEY_LEFTALT!() | input_event_codes::KEY_RIGHTALT!() => mods |= 8,
+				input_event_codes::KEY_LEFTMETA!() | input_event_codes::KEY_RIGHTMETA!() => {
+					mods |= 64
+				}
+				input_event_codes::KEY_CAPSLOCK!() => self.mods_locked ^= 1,
+				_ => {}
+			}
+		}
+
+		self.mods_depressed = mods;
+	}
+}
+
+#[derive(Dispatcher)]
+pub struct Keyboard {
+	pub id: ObjectId,
+	focused_surface: Mutex<Weak<Surface>>,
+	modifier_state: Mutex<ModifierState>,
+	pressed_keys: DashMap<ObjectId, DashSet<u32>>,
+	current_keymap_id: Mutex<u64>,
+}
+
+impl Keyboard {
+	pub fn new(id: ObjectId) -> Self {
+		Self {
+			id,
+			focused_surface: Mutex::new(Weak::new()),
+			modifier_state: Mutex::new(ModifierState::default()),
+			pressed_keys: DashMap::default(),
+			current_keymap_id: Mutex::new(0),
+		}
+	}
+
+	async fn send_keymap(&self, client: &mut Client, keymap: &[u8]) -> Result<()> {
+		let mut file = MemfdOptions::default()
+			.create("stardust-keymap")
+			.map_err(|e| waynest::server::Error::Custom(e.to_string()))?
+			.into_file();
+		file.set_len(keymap.len() as u64)?;
+		file.write_all(keymap)?;
+		file.flush()?;
+
+		let fd = unsafe { OwnedFd::from_raw_fd(file.into_raw_fd()) };
+
+		// Send keymap to client
+		self.keymap(
+			client,
+			self.id,
+			KeymapFormat::XkbV1,
+			fd,
+			keymap.len() as u32,
+		)
+		.await?;
+
+		Ok(())
+	}
+
+	/// has to be the wayland key, so -8 or whatever
+	pub async fn handle_keyboard_key(
+		&self,
+		client: &mut Client,
+		surface: Arc<Surface>,
+		keymap_id: u64,
+		key: u32,
+		pressed: bool,
+	) -> Result<()> {
+		// KEYMAP UPDATES
+		{
+			let mut old_keymap_id = self.current_keymap_id.lock().await;
+
+			if *old_keymap_id != keymap_id {
+				let keymap_key = DefaultKey::from(KeyData::from_ffi(keymap_id));
+
+				// Get keymap data and drop the lock immediately
+				let keymap_data = {
+					let keymap_lock = KEYMAPS.lock();
+					keymap_lock
+						.get(keymap_key)
+						.map(|s| s.as_bytes().to_vec())
+						.unwrap_or_default()
+				};
+
+				// Now we can safely await
+				self.send_keymap(client, &keymap_data).await?;
+			};
+			*old_keymap_id = keymap_id;
+			drop(old_keymap_id);
+		}
+
+		// PRESSED KEYS UPDATE
+		let pressed_keys = self.pressed_keys.entry(surface.id).or_default();
+		if pressed {
+			pressed_keys.insert(key);
+		} else {
+			pressed_keys.remove(&key);
+		}
+		// println!("pressed keys: {:?}", &*pressed_keys);
+
+		// FOCUS UPDATES
+		let mut focused = self.focused_surface.lock().await;
+		let mut modifier_state = self.modifier_state.lock().await;
+
+		let refocus = focused.as_ptr() != Arc::as_ptr(&surface);
+		// If we're entering a new surface
+		if refocus {
+			// Send leave to old surface if it exists and is still alive
+			if let Some(old_surface) = focused.upgrade() {
+				let serial = client.next_event_serial();
+				self.leave(client, old_surface.id, serial, self.id).await?;
+				// println!("Left surface {}", old_surface.id);
+			}
+
+			// Send enter to new surface
+			let serial = client.next_event_serial();
+			self.enter(
+				client,
+				self.id,
+				serial,
+				surface.id,
+				pressed_keys.iter().flat_map(|k| k.to_ne_bytes()).collect(),
+			)
+			.await?;
+			// println!("Entered new surface {}", surface.id);
+
+			// Update focused surface
+			*focused = Arc::downgrade(&surface);
+		}
+
+		// KEY EVENT SENDING
+		let serial = client.next_event_serial();
+		// println!(
+		// 	"Sent key {key} {}",
+		// 	if pressed { "pressed" } else { "released" }
+		// );
+		self.key(
+			client,
+			self.id,
+			serial,
+			client.display().creation_time.elapsed().as_millis() as u32, // time
+			key,
+			if pressed {
+				KeyState::Pressed
+			} else {
+				KeyState::Released
+			},
+		)
+		.await?;
+
+		// MODIFIER UPDATES
+		// Update modifier state and send modifiers event if changed
+		if refocus || modifier_state.update_key(key, pressed) {
+			// println!("Update modifiers");
+			let serial = client.next_event_serial();
+			self.modifiers(
+				client,
+				self.id,
+				serial,
+				modifier_state.mods_depressed,
+				modifier_state.mods_latched,
+				modifier_state.mods_locked,
+				modifier_state.group,
+			)
+			.await?;
+		}
+
+		Ok(())
+	}
+
+	pub async fn reset(&self, client: &mut Client) -> Result<()> {
+		let mut modifier_state = self.modifier_state.lock().await;
+		modifier_state.pressed_keys.clear();
+		modifier_state.mods_depressed = 0;
+		modifier_state.mods_latched = 0;
+		modifier_state.mods_locked = 0;
+		modifier_state.group = 0;
+
+		let serial = client.next_event_serial();
+		self.modifiers(
+			client,
+			self.id,
+			serial,
+			modifier_state.mods_depressed,
+			modifier_state.mods_latched,
+			modifier_state.mods_locked,
+			modifier_state.group,
+		)
+		.await
+	}
+}
+
+impl WlKeyboard for Keyboard {
+	/// https://wayland.app/protocols/wayland#wl_keyboard:request:release
+	async fn release(&self, _client: &mut Client, _sender_id: ObjectId) -> Result<()> {
+		Ok(())
+	}
+}

src/wayland/core/mod.rs

@@ -0,0 +1,13 @@
+pub mod buffer;
+pub mod callback;
+pub mod compositor;
+pub mod data_device;
+pub mod keyboard;
+pub mod output;
+pub mod pointer;
+pub mod seat;
+pub mod shm;
+pub mod shm_buffer_backing;
+pub mod shm_pool;
+pub mod surface;
+pub mod touch;

src/wayland/core/output.rs

@@ -0,0 +1,43 @@
+use waynest::{
+	server::{Client, Dispatcher, Result},
+	wire::ObjectId,
+};
+
+pub use waynest::server::protocol::core::wayland::wl_output::*;
+
+#[derive(Debug, Dispatcher)]
+pub struct Output {
+	pub id: ObjectId,
+	pub version: u32,
+}
+impl Output {
+	pub async fn advertise_outputs(&self, client: &mut Client) -> Result<()> {
+		self.geometry(
+			client,
+			self.id,
+			2048,
+			2048,
+			0,
+			0,
+			Subpixel::None,
+			"Stardust Virtual Display".to_string(),
+			"Stardust Virtual Display".to_string(),
+			Transform::Normal,
+		)
+		.await?;
+
+		self.mode(client, self.id, Mode::Current, 2048, 2048, i32::MAX)
+			.await?;
+
+		if self.version >= 2 {
+			self.done(client, self.id).await?;
+		}
+		Ok(())
+	}
+}
+impl WlOutput for Output {
+	/// https://wayland.app/protocols/wayland#wl_output:request:release
+	async fn release(&self, _client: &mut Client, _sender_id: ObjectId) -> Result<()> {
+		Ok(())
+	}
+}

src/wayland/core/pointer.rs

@@ -0,0 +1,190 @@
+use crate::wayland::core::{seat::fixed_from_f32, surface::Surface};
+use mint::Vector2;
+use std::sync::Arc;
+use std::sync::Weak;
+use tokio::sync::Mutex;
+use tracing;
+pub use waynest::server::protocol::core::wayland::wl_pointer::*;
+use waynest::{
+	server::{Client, Dispatcher, Result},
+	wire::ObjectId,
+};
+
+#[derive(Dispatcher)]
+pub struct Pointer {
+	pub id: ObjectId,
+	version: u32,
+	focused_surface: Mutex<Weak<Surface>>,
+}
+impl Pointer {
+	pub fn new(id: ObjectId, version: u32) -> Self {
+		Self {
+			id,
+			version,
+			focused_surface: Mutex::new(Weak::new()),
+		}
+	}
+
+	pub async fn handle_pointer_motion(
+		&self,
+		client: &mut Client,
+		surface: Arc<Surface>,
+		position: Vector2<f32>,
+	) -> Result<()> {
+		tracing::debug!(
+			"Handling pointer motion at ({}, {})",
+			position.x,
+			position.y
+		);
+		let mut focused = self.focused_surface.lock().await;
+
+		// If we're entering a new surface
+		if focused.as_ptr() != Arc::as_ptr(&surface) {
+			tracing::debug!("Surface transition detected");
+			// Send leave to old surface if it exists and is still alive
+			if let Some(old_surface) = focused.upgrade() {
+				let serial = client.next_event_serial();
+				tracing::debug!("Sending leave event with serial {}", serial);
+				self.leave(client, self.id, serial, old_surface.id).await?;
+			}
+
+			// Send enter to new surface
+			let serial = client.next_event_serial();
+			tracing::debug!(
+				"Sending enter event with serial {} to surface {:?}",
+				serial,
+				surface.id
+			);
+			self.enter(
+				client,
+				self.id,
+				serial,
+				surface.id,
+				fixed_from_f32(position.x),
+				fixed_from_f32(position.y),
+			)
+			.await?;
+
+			// Update focused surface
+			*focused = Arc::downgrade(&surface);
+		}
+
+		// Send motion event to current surface
+		tracing::debug!("Sending motion event to surface");
+		self.motion(
+			client,
+			self.id,
+			0, // time
+			fixed_from_f32(position.x),
+			fixed_from_f32(position.y),
+		)
+		.await?;
+		if self.version >= 5 {
+			self.frame(client, self.id).await?;
+		}
+
+		Ok(())
+	}
+
+	pub async fn handle_pointer_button(
+		&self,
+		client: &mut Client,
+		surface: Arc<Surface>,
+		button: u32,
+		pressed: bool,
+	) -> Result<()> {
+		tracing::debug!(
+			"Handling pointer button {} {} on surface {:?}",
+			button,
+			if pressed { "pressed" } else { "released" },
+			surface.id
+		);
+		let serial = client.next_event_serial();
+		self.button(
+			client,
+			self.id,
+			serial,
+			0, // time
+			button,
+			if pressed {
+				ButtonState::Pressed
+			} else {
+				ButtonState::Released
+			},
+		)
+		.await?;
+		self.frame(client, self.id).await
+	}
+	pub async fn handle_pointer_scroll(
+		&self,
+		client: &mut Client,
+		_surface: Arc<Surface>,
+		scroll_distance: Option<Vector2<f32>>,
+		scroll_steps: Option<Vector2<f32>>,
+	) -> Result<()> {
+		tracing::debug!(
+			"Handling pointer scroll: distance={:?}, steps={:?}",
+			scroll_distance,
+			scroll_steps
+		);
+		if let Some(distance) = scroll_distance {
+			self.axis(
+				client,
+				self.id,
+				0, // time
+				Axis::HorizontalScroll,
+				fixed_from_f32(distance.x),
+			)
+			.await?;
+			self.axis(
+				client,
+				self.id,
+				0, // time
+				Axis::VerticalScroll,
+				fixed_from_f32(distance.y),
+			)
+			.await?;
+		}
+		if self.version >= 5 {
+			if let Some(steps) = scroll_steps {
+				self.axis_discrete(client, self.id, Axis::HorizontalScroll, steps.x as i32)
+					.await?;
+				self.axis_discrete(client, self.id, Axis::VerticalScroll, steps.y as i32)
+					.await?;
+			}
+			self.frame(client, self.id).await?;
+		}
+		Ok(())
+	}
+
+	pub async fn reset(&self, client: &mut Client) -> Result<()> {
+		let mut focused = self.focused_surface.lock().await;
+		if let Some(old_surface) = focused.upgrade() {
+			let serial = client.next_event_serial();
+			self.leave(client, self.id, serial, old_surface.id).await?;
+			self.frame(client, self.id).await?;
+		}
+		*focused = Weak::new();
+		Ok(())
+	}
+}
+
+impl WlPointer for Pointer {
+	/// https://wayland.app/protocols/wayland#wl_pointer:request:set_cursor
+	async fn set_cursor(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_serial: u32,
+		_surface: Option<ObjectId>,
+		_hotspot_x: i32,
+		_hotspot_y: i32,
+	) -> Result<()> {
+		Ok(())
+	}
+
+	/// https://wayland.app/protocols/wayland#wl_pointer:request:release
+	async fn release(&self, _client: &mut Client, _sender_id: ObjectId) -> Result<()> {
+		Ok(())
+	}
+}

src/wayland/core/seat.rs

@@ -0,0 +1,200 @@
+use crate::wayland::core::{keyboard::Keyboard, pointer::Pointer, surface::Surface, touch::Touch};
+use mint::Vector2;
+use std::sync::Arc;
+use std::sync::OnceLock;
+pub use waynest::server::protocol::core::wayland::wl_seat::*;
+use waynest::server::{Client, Dispatcher, Result};
+use waynest::wire::{Fixed, ObjectId};
+
+#[derive(Debug)]
+pub enum SeatMessage {
+	PointerMotion {
+		surface: Arc<Surface>,
+		position: Vector2<f32>,
+	},
+	PointerButton {
+		surface: Arc<Surface>,
+		button: u32,
+		pressed: bool,
+	},
+	PointerScroll {
+		surface: Arc<Surface>,
+		scroll_distance: Option<Vector2<f32>>,
+		scroll_steps: Option<Vector2<f32>>,
+	},
+	KeyboardKey {
+		surface: Arc<Surface>,
+		keymap_id: u64,
+		key: u32,
+		pressed: bool,
+	},
+	TouchDown {
+		surface: Arc<Surface>,
+		id: u32,
+		position: Vector2<f32>,
+	},
+	TouchMove {
+		id: u32,
+		position: Vector2<f32>,
+	},
+	TouchUp {
+		id: u32,
+	},
+	Reset,
+}
+
+pub fn fixed_from_f32(f: f32) -> Fixed {
+	unsafe { Fixed::from_raw((f * 256.0).round() as u32) }
+}
+
+#[derive(Default, Dispatcher)]
+pub struct Seat {
+	version: u32,
+	pointer: OnceLock<Arc<Pointer>>,
+	keyboard: OnceLock<Arc<Keyboard>>,
+	touch: OnceLock<Arc<Touch>>,
+}
+
+impl Seat {
+	pub async fn new(client: &mut Client, id: ObjectId, version: u32) -> Result<Self> {
+		let seat = Self {
+			version,
+			pointer: OnceLock::new(),
+			keyboard: OnceLock::new(),
+			touch: OnceLock::new(),
+		};
+
+		if version >= 2 {
+			seat.name(client, id, "theonlyseat".into()).await?;
+		}
+
+		tracing::debug!("Advertising seat capabilities with id {}", id);
+		let capabilities = Capability::Pointer | Capability::Keyboard | Capability::Touch;
+		WlSeat::capabilities(&seat, client, id, capabilities).await?;
+		tracing::debug!("Capabilities advertised: {:?}", capabilities);
+
+		Ok(seat)
+	}
+
+	pub async fn handle_message(&self, client: &mut Client, message: SeatMessage) -> Result<()> {
+		match message {
+			SeatMessage::PointerMotion { surface, position } => {
+				if let Some(pointer) = self.pointer.get() {
+					pointer
+						.handle_pointer_motion(client, surface, position)
+						.await?;
+				}
+			}
+			SeatMessage::PointerButton {
+				surface,
+				button,
+				pressed,
+			} => {
+				if let Some(pointer) = self.pointer.get() {
+					pointer
+						.handle_pointer_button(client, surface, button, pressed)
+						.await?;
+				}
+			}
+			SeatMessage::PointerScroll {
+				surface,
+				scroll_distance,
+				scroll_steps,
+			} => {
+				if let Some(pointer) = self.pointer.get() {
+					pointer
+						.handle_pointer_scroll(client, surface, scroll_distance, scroll_steps)
+						.await?;
+				}
+			}
+			SeatMessage::KeyboardKey {
+				surface,
+				keymap_id,
+				key,
+				pressed,
+			} => {
+				if let Some(keyboard) = self.keyboard.get() {
+					keyboard
+						.handle_keyboard_key(client, surface, keymap_id, key - 8, pressed)
+						.await?;
+				}
+			}
+			SeatMessage::TouchDown {
+				surface,
+				id,
+				position,
+			} => {
+				if let Some(touch) = self.touch.get() {
+					touch
+						.handle_touch_down(client, surface, id, position)
+						.await?;
+				}
+			}
+			SeatMessage::TouchMove { id, position } => {
+				if let Some(touch) = self.touch.get() {
+					touch.handle_touch_move(client, id, position).await?;
+				}
+			}
+			SeatMessage::TouchUp { id } => {
+				if let Some(touch) = self.touch.get() {
+					touch.handle_touch_up(client, id).await?;
+				}
+			}
+			SeatMessage::Reset => {
+				if let Some(pointer) = self.pointer.get() {
+					pointer.reset(client).await?;
+				}
+				if let Some(keyboard) = self.keyboard.get() {
+					keyboard.reset(client).await?;
+				}
+				if let Some(touch) = self.touch.get() {
+					touch.reset(client).await?;
+				}
+			}
+		}
+		Ok(())
+	}
+}
+impl WlSeat for Seat {
+	/// https://wayland.app/protocols/wayland#wl_seat:request:get_pointer
+	async fn get_pointer(
+		&self,
+		client: &mut Client,
+		_sender_id: ObjectId,
+		id: ObjectId,
+	) -> Result<()> {
+		let pointer = client.insert(id, Pointer::new(id, self.version));
+		let _ = self.pointer.set(pointer);
+		Ok(())
+	}
+
+	/// https://wayland.app/protocols/wayland#wl_seat:request:get_keyboard
+	async fn get_keyboard(
+		&self,
+		client: &mut Client,
+		_sender_id: ObjectId,
+		id: ObjectId,
+	) -> Result<()> {
+		tracing::info!("Getting keyboard");
+		let keyboard = client.insert(id, Keyboard::new(id));
+		let _ = self.keyboard.set(keyboard);
+		Ok(())
+	}
+
+	/// https://wayland.app/protocols/wayland#wl_seat:request:get_touch
+	async fn get_touch(
+		&self,
+		client: &mut Client,
+		_sender_id: ObjectId,
+		id: ObjectId,
+	) -> Result<()> {
+		let touch = client.insert(id, Touch(id));
+		let _ = self.touch.set(touch);
+		Ok(())
+	}
+
+	/// https://wayland.app/protocols/wayland#wl_seat:request:release
+	async fn release(&self, _client: &mut Client, _sender_id: ObjectId) -> Result<()> {
+		Ok(())
+	}
+}

src/wayland/core/shm.rs

@@ -0,0 +1,39 @@
+use std::os::fd::OwnedFd;
+
+use crate::wayland::core::shm_pool::ShmPool;
+pub use waynest::server::protocol::core::wayland::wl_shm::*;
+use waynest::{
+	server::{Client, Dispatcher, Result},
+	wire::ObjectId,
+};
+
+#[derive(Debug, Dispatcher, Default)]
+pub struct Shm;
+impl Shm {
+	pub async fn advertise_formats(&self, client: &mut Client, sender_id: ObjectId) -> Result<()> {
+		self.format(client, sender_id, Format::Argb8888).await?;
+		self.format(client, sender_id, Format::Xrgb8888).await?;
+
+		Ok(())
+	}
+}
+impl WlShm for Shm {
+	/// https://wayland.app/protocols/wayland#wl_shm:request:create_pool
+	async fn create_pool(
+		&self,
+		client: &mut Client,
+		_sender_id: ObjectId,
+		pool_id: ObjectId,
+		fd: OwnedFd,
+		size: i32,
+	) -> Result<()> {
+		client.insert(pool_id, ShmPool::new(fd, size)?);
+
+		Ok(())
+	}
+
+	/// https://wayland.app/protocols/wayland#wl_shm:request:release
+	async fn release(&self, _client: &mut Client, _sender_id: ObjectId) -> Result<()> {
+		Ok(())
+	}
+}

src/wayland/core/shm_buffer_backing.rs

@@ -0,0 +1,99 @@
+use super::shm_pool::ShmPool;
+use bevy::{
+	asset::{Assets, Handle, RenderAssetUsages},
+	image::Image,
+	render::render_resource::{Extent3d, TextureDimension, TextureFormat},
+};
+use mint::Vector2;
+use std::sync::Arc;
+use waynest::server::protocol::core::wayland::wl_shm::Format;
+/// Parameters for a shared memory buffer
+#[derive(Debug)]
+pub struct ShmBufferBacking {
+	pool: Arc<ShmPool>,
+	offset: usize,
+	stride: usize,
+	size: Vector2<usize>,
+	format: Format,
+}
+
+impl ShmBufferBacking {
+	pub fn new(
+		pool: Arc<ShmPool>,
+		offset: usize,
+		stride: usize,
+		size: Vector2<usize>,
+		format: Format,
+	) -> Self {
+		Self {
+			pool,
+			offset,
+			stride,
+			size,
+			format,
+		}
+	}
+
+	#[tracing::instrument("debug", skip_all)]
+	pub fn update_tex(&self, images: &mut Assets<Image>) -> Option<Handle<Image>> {
+		let src_data_lock = self.pool.data_lock();
+		let mut src_cursor = self.offset;
+
+		// Calculate maximum cursor position needed - stride is already in bytes
+		let max_cursor = self.offset + (self.size.y * self.stride);
+
+		// Check if we have enough data
+		if max_cursor > src_data_lock.len() {
+			return None;
+		}
+		let data_len = self.size.x * self.size.y * 4;
+		if src_data_lock.len() != data_len {
+			return None;
+		}
+		let mut dst_cursor = 0;
+
+		let mut dst_data = vec![0u8; data_len];
+		for _y in 0..self.size.y {
+			for _x in 0..self.size.x {
+				match self.format {
+					Format::Argb8888 | Format::Xrgb8888 => {
+						dst_data[dst_cursor] = src_data_lock[src_cursor + 2]; // Red is byte 2
+						dst_data[dst_cursor + 1] = src_data_lock[src_cursor + 1]; // Green is byte 1
+						dst_data[dst_cursor + 2] = src_data_lock[src_cursor]; // Blue is byte 0
+						dst_data[dst_cursor + 3] = src_data_lock[src_cursor + 3]; // Alpha is byte 3
+					}
+					_ => panic!("Unsupported format {:?}", self.format),
+				}
+				src_cursor += 4;
+				dst_cursor += 4;
+			}
+			src_cursor += self.stride - (self.size.x * 4);
+		}
+
+		let image = Image::new(
+			Extent3d {
+				width: self.size().x as u32,
+				height: self.size().y as u32,
+				depth_or_array_layers: 1,
+			},
+			TextureDimension::D2,
+			dst_data,
+			TextureFormat::Rgba8UnormSrgb,
+			RenderAssetUsages::RENDER_WORLD,
+		);
+
+		Some(images.add(image))
+	}
+
+	pub fn is_transparent(&self) -> bool {
+		match self.format {
+			Format::Xrgb8888 => false,
+			Format::Argb8888 => true,
+			_ => true,
+		}
+	}
+
+	pub fn size(&self) -> Vector2<usize> {
+		self.size
+	}
+}

src/wayland/core/shm_pool.rs

@@ -0,0 +1,79 @@
+use memmap2::{MmapOptions, RemapOptions};
+use parking_lot::{Mutex, MutexGuard, RawMutex, lock_api::MappedMutexGuard};
+use std::os::fd::{IntoRawFd, OwnedFd};
+use waynest::{
+	server::{Client, Dispatcher, Result, protocol::core::wayland::wl_shm::Format},
+	wire::ObjectId,
+};
+
+use crate::wayland::core::buffer::{Buffer, BufferBacking};
+
+pub use waynest::server::protocol::core::wayland::wl_shm_pool::*;
+
+use super::shm_buffer_backing::ShmBufferBacking;
+
+#[derive(Debug, Dispatcher)]
+pub struct ShmPool {
+	inner: Mutex<memmap2::MmapMut>,
+}
+
+impl ShmPool {
+	#[tracing::instrument(level = "debug", skip_all)]
+	pub fn new(fd: OwnedFd, size: i32) -> Result<Self> {
+		let map = unsafe {
+			MmapOptions::new()
+				.len(size as usize)
+				.map_mut(fd.into_raw_fd())?
+		};
+
+		Ok(Self {
+			inner: Mutex::new(map),
+		})
+	}
+
+	#[tracing::instrument(level = "debug", skip_all)]
+	pub fn data_lock(&self) -> MappedMutexGuard<RawMutex, [u8]> {
+		MutexGuard::map(self.inner.lock(), |i| i.as_mut())
+	}
+}
+
+impl WlShmPool for ShmPool {
+	/// https://wayland.app/protocols/wayland#wl_shm_pool:request:create_buffer
+	#[tracing::instrument(level = "debug", skip_all)]
+	async fn create_buffer(
+		&self,
+		client: &mut Client,
+		sender_id: ObjectId,
+		id: ObjectId,
+		offset: i32,
+		width: i32,
+		height: i32,
+		stride: i32,
+		format: Format,
+	) -> Result<()> {
+		let params = ShmBufferBacking::new(
+			client.get::<ShmPool>(sender_id).unwrap(),
+			offset as usize,
+			stride as usize,
+			[width as usize, height as usize].into(),
+			format,
+		);
+
+		Buffer::new(client, id, BufferBacking::Shm(params));
+		Ok(())
+	}
+
+	/// https://wayland.app/protocols/wayland#wl_shm_pool:request:resize
+	#[tracing::instrument(level = "debug", skip_all)]
+	async fn resize(&self, _client: &mut Client, _sender_id: ObjectId, size: i32) -> Result<()> {
+		let mut inner = self.inner.lock();
+		unsafe { inner.remap(size as usize, RemapOptions::new().may_move(true))? };
+		Ok(())
+	}
+
+	/// https://wayland.app/protocols/wayland#wl_shm_pool:request:destroy
+	#[tracing::instrument(level = "debug", skip_all)]
+	async fn destroy(&self, _client: &mut Client, _sender_id: ObjectId) -> Result<()> {
+		Ok(())
+	}
+}

src/wayland/core/surface.rs

@@ -0,0 +1,426 @@
+use super::{buffer::Buffer, callback::Callback};
+use crate::{
+	BevyMaterial,
+	core::registry::Registry,
+	nodes::{drawable::model::ModelPart, items::panel::Geometry},
+	wayland::{
+		Message, MessageSink,
+		core::buffer::BufferUsage,
+		presentation::{MonotonicTimestamp, PresentationFeedback},
+		util::{ClientExt, DoubleBuffer},
+		xdg::{popup::Popup, toplevel::Toplevel},
+	},
+};
+use bevy::{
+	asset::{Assets, Handle},
+	image::Image,
+	render::alpha::AlphaMode,
+};
+use bevy_dmabuf::import::ImportedDmatexs;
+use mint::Vector2;
+use parking_lot::Mutex;
+use std::sync::{Arc, OnceLock};
+use waynest::{
+	server::{
+		Client, Dispatcher, Result,
+		protocol::{
+			core::wayland::{wl_output::Transform, wl_surface::*},
+			stable::presentation_time::wp_presentation_feedback::{Kind, WpPresentationFeedback},
+		},
+	},
+	wire::ObjectId,
+};
+
+pub static WL_SURFACE_REGISTRY: Registry<Surface> = Registry::new();
+
+#[derive(Debug, Clone)]
+pub enum SurfaceRole {
+	XdgToplevel(Arc<Toplevel>),
+	XDGPopup(Arc<Popup>),
+}
+
+#[derive(Debug, Clone)]
+pub struct BufferState {
+	pub buffer: Arc<Buffer>,
+	pub usage: Option<Arc<BufferUsage>>,
+}
+
+#[derive(Debug, Clone)]
+pub struct SurfaceState {
+	pub buffer: Option<BufferState>,
+	pub density: f32,
+	pub geometry: Option<Geometry>,
+	pub min_size: Option<Vector2<u32>>,
+	pub max_size: Option<Vector2<u32>>,
+	frame_callbacks: Vec<Arc<Callback>>,
+}
+impl Default for SurfaceState {
+	fn default() -> Self {
+		Self {
+			buffer: Default::default(),
+			density: 1.0,
+			geometry: None,
+			min_size: None,
+			max_size: None,
+			frame_callbacks: Vec::new(),
+		}
+	}
+}
+
+// if returning false, don't run this callback again... just remove it
+pub type OnCommitCallback = Box<dyn Fn(&Surface, &SurfaceState) -> bool + Send + Sync>;
+
+#[derive(Dispatcher)]
+pub struct Surface {
+	pub id: ObjectId,
+	state: Mutex<DoubleBuffer<SurfaceState>>,
+	pub message_sink: MessageSink,
+	// TODO: This should probably be a OnceLock? wayland doesn't support changing the surface role
+	pub role: Mutex<Option<SurfaceRole>>,
+	on_commit_handlers: Mutex<Vec<OnCommitCallback>>,
+	material: OnceLock<Handle<BevyMaterial>>,
+	pending_material_applications: Registry<ModelPart>,
+	presentation_feedback: Mutex<Vec<Arc<PresentationFeedback>>>,
+}
+impl std::fmt::Debug for Surface {
+	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+		f.debug_struct("Surface")
+			.field("state", &self.state)
+			.field("message_sink", &self.message_sink)
+			.field("role", &self.role)
+			.field(
+				"on_commit_handlers",
+				&format!("<{} handlers>", self.on_commit_handlers.lock().len()),
+			)
+			.finish()
+	}
+}
+impl Surface {
+	#[tracing::instrument(level = "debug", skip_all)]
+	pub fn new(client: &Client, id: ObjectId) -> Self {
+		Surface {
+			id,
+			state: Default::default(),
+			message_sink: client.message_sink(),
+			role: Mutex::new(None),
+			on_commit_handlers: Mutex::new(Vec::new()),
+			material: OnceLock::new(),
+			pending_material_applications: Registry::new(),
+			presentation_feedback: Mutex::default(),
+		}
+	}
+
+	#[tracing::instrument(level = "debug", skip_all)]
+	pub fn pending_state(&self) -> parking_lot::MutexGuard<'_, DoubleBuffer<SurfaceState>> {
+		self.state.lock()
+	}
+
+	#[tracing::instrument(level = "debug", skip_all)]
+	pub fn add_commit_handler<F: Fn(&Surface, &SurfaceState) -> bool + Send + Sync + 'static>(
+		&self,
+		handler: F,
+	) {
+		let mut handlers = self.on_commit_handlers.lock();
+		handlers.push(Box::new(handler));
+	}
+
+	#[tracing::instrument(level = "debug", skip_all)]
+	pub fn update_graphics(
+		&self,
+		dmatexes: &ImportedDmatexs,
+		materials: &mut Assets<BevyMaterial>,
+		images: &mut Assets<Image>,
+	) {
+		let Some(buffer) = self.state.lock().current().buffer.clone() else {
+			return;
+		};
+
+		let material = self.material.get_or_init(|| {
+			// // Set default shader parameters
+			// let mut params = mat_wrapper.0.get_all_param_info();
+			// params.set_vec2("uv_scale", stereokit_rust::maths::Vec2::new(1.0, 1.0));
+			// params.set_vec2("uv_offset", stereokit_rust::maths::Vec2::new(0.0, 0.0));
+			// params.set_float("fcFactor", 1.0);
+			// params.set_float("ripple", 4.0);
+			// params.set_float("alpha_min", 0.0);
+			// params.set_float("alpha_max", 1.0);
+
+			materials.add(BevyMaterial {
+				unlit: true,
+				..Default::default()
+			})
+		});
+
+		if let Some(new_tex) = buffer.buffer.update_tex(dmatexes, images) {
+			let material = materials.get_mut(material).unwrap();
+			material.base_color_texture.replace(new_tex);
+			material.alpha_mode = if buffer.buffer.is_transparent() {
+				AlphaMode::Premultiplied
+			} else {
+				AlphaMode::Opaque
+			};
+		}
+
+		self.apply_surface_materials();
+	}
+
+	#[tracing::instrument(level = "debug", skip_all)]
+	pub fn apply_material(&self, model_part: &Arc<ModelPart>) {
+		// tracing::info!("uwu applying material");
+		self.pending_material_applications.add_raw(model_part)
+	}
+
+	#[tracing::instrument(level = "debug", skip_all)]
+	fn apply_surface_materials(&self) {
+		let Some(mat) = self.material.get() else {
+			return;
+		};
+
+		for model_node in self.pending_material_applications.get_valid_contents() {
+			model_node.replace_material(mat.clone());
+		}
+		self.pending_material_applications.clear();
+	}
+	#[tracing::instrument("debug", skip_all)]
+	pub fn current_state(&self) -> SurfaceState {
+		self.state.lock().current().clone()
+	}
+	#[tracing::instrument(level = "debug", skip_all)]
+	pub fn frame_event(&self) {
+		for callback in self.state.lock().current.frame_callbacks.drain(..) {
+			let _ = self.message_sink.send(Message::Frame(callback));
+		}
+	}
+	// pub fn size(&self) -> Option<Vector2<u32>> {
+	// 	self.state
+	// 		.lock()
+	// 		.current()
+	// 		.buffer
+	// 		.as_ref()
+	// 		.map(|b| [b.size.x as u32, b.size.y as u32].into())
+	// }
+
+	// pub async fn release_old_buffer(&self, client: &mut Client) -> Result<()> {
+	// 	let (old_buffer, object) = {
+	// 		let lock = self.state.lock();
+
+	// 		let Some(old_buffer) = lock.current().buffer.clone() else {
+	// 			return Ok(());
+	// 		};
+	// 		let new_buffer = lock.pending.buffer.as_ref();
+	// 		if new_buffer.map(Arc::as_ptr) == Some(Arc::as_ptr(&old_buffer)) {
+	// 			return Ok(());
+	// 		}
+	// 		drop(lock);
+
+	// 		(old_buffer.clone(), old_buffer.id)
+	// 	};
+	// 	old_buffer.release(client, object).await?;
+
+	// 	Ok(())
+	// }
+
+	#[tracing::instrument(level = "debug", skip_all)]
+	pub fn add_presentation_feedback(&self, feedback: Arc<PresentationFeedback>) {
+		self.presentation_feedback.lock().push(feedback);
+	}
+
+	pub fn submit_presentation_feedback(
+		self: &Arc<Self>,
+		display_timestamp: MonotonicTimestamp,
+		refresh_cycle: u64,
+	) {
+		self.message_sink
+			.send(Message::SendPresentationFeedback {
+				surface: self.clone(),
+				display_timestamp,
+				refresh_cycle,
+			})
+			.unwrap();
+	}
+
+	#[tracing::instrument(level = "debug", skip_all)]
+	pub async fn send_presentation_feedback(
+		&self,
+		client: &mut Client,
+		display_timestamp: MonotonicTimestamp,
+		refresh_cycle: u64,
+	) -> Result<()> {
+		let feedbacks = self
+			.presentation_feedback
+			.lock()
+			.drain(..)
+			.collect::<Vec<_>>();
+		for feedback in feedbacks {
+			feedback
+				.sync_output(
+					client,
+					feedback.0,
+					client.display().output.get().unwrap().id,
+				)
+				.await?;
+			let cycle_lo = refresh_cycle as u32;
+			let cycle_hi = (refresh_cycle >> 32) as u32;
+			feedback
+				.presented(
+					client,
+					feedback.0,
+					display_timestamp.secs_hi(),
+					display_timestamp.secs_lo(),
+					display_timestamp.subsec_nanos(),
+					0,
+					cycle_hi,
+					cycle_lo,
+					Kind::empty(),
+				)
+				.await?;
+		}
+		Ok(())
+	}
+}
+impl WlSurface for Surface {
+	/// https://wayland.app/protocols/wayland#wl_surface:request:attach
+	#[tracing::instrument(level = "debug", skip_all)]
+	async fn attach(
+		&self,
+		client: &mut Client,
+		_sender_id: ObjectId,
+		buffer: Option<ObjectId>,
+		_x: i32,
+		_y: i32,
+	) -> Result<()> {
+		self.state.lock().pending.buffer = buffer.and_then(|b| {
+			let buffer = client.get::<Buffer>(b)?;
+			let mut usage = Some(BufferUsage::new(client, &buffer));
+			Some(BufferState {
+				usage: usage.take_if(|_| buffer.uses_buffer_usage()),
+				buffer,
+			})
+		});
+		Ok(())
+	}
+
+	/// https://wayland.app/protocols/wayland#wl_surface:request:damage
+	#[tracing::instrument(level = "debug", skip_all)]
+	async fn damage(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_x: i32,
+		_y: i32,
+		_width: i32,
+		_height: i32,
+	) -> Result<()> {
+		Ok(())
+	}
+
+	/// https://wayland.app/protocols/wayland#wl_surface:request:frame
+	#[tracing::instrument(level = "debug", skip_all)]
+	async fn frame(
+		&self,
+		client: &mut Client,
+		_sender_id: ObjectId,
+		callback_id: ObjectId,
+	) -> Result<()> {
+		let callback = client.insert(callback_id, Callback(callback_id));
+		self.state.lock().pending.frame_callbacks.push(callback);
+		Ok(())
+	}
+
+	/// https://wayland.app/protocols/wayland#wl_surface:request:set_opaque_region
+	#[tracing::instrument(level = "debug", skip_all)]
+	async fn set_opaque_region(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_region: Option<ObjectId>,
+	) -> Result<()> {
+		// nothing we can really do to repaint behind this so ignore it
+		Ok(())
+	}
+
+	/// https://wayland.app/protocols/wayland#wl_surface:request:set_input_region
+	#[tracing::instrument(level = "debug", skip_all)]
+	async fn set_input_region(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_region: Option<ObjectId>,
+	) -> Result<()> {
+		// too complicated to implement this for now so who the hell cares
+		Ok(())
+	}
+
+	/// https://wayland.app/protocols/wayland#wl_surface:request:commit
+	#[tracing::instrument(level = "debug", skip_all)]
+	async fn commit(&self, _client: &mut Client, _sender_id: ObjectId) -> Result<()> {
+		self.state.lock().apply();
+		self.state.lock().pending.frame_callbacks.clear();
+		let current_state = self.current_state();
+		let mut handlers = self.on_commit_handlers.lock();
+		handlers.retain(|f| (f)(self, &current_state));
+		Ok(())
+	}
+
+	/// https://wayland.app/protocols/wayland#wl_surface:request:set_buffer_transform
+	#[tracing::instrument(level = "debug", skip_all)]
+	async fn set_buffer_transform(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_transform: Transform,
+	) -> Result<()> {
+		// we just don't have the output transform or fullscreen at all so this optimization is never needed
+		Ok(())
+	}
+
+	/// https://wayland.app/protocols/wayland#wl_surface:request:set_buffer_scale
+	#[tracing::instrument(level = "debug", skip_all)]
+	async fn set_buffer_scale(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		scale: i32,
+	) -> Result<()> {
+		self.state.lock().pending.density = scale as f32;
+		Ok(())
+	}
+
+	/// https://wayland.app/protocols/wayland#wl_surface:request:damage_buffer
+	#[tracing::instrument(level = "debug", skip_all)]
+	async fn damage_buffer(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_x: i32,
+		_y: i32,
+		_width: i32,
+		_height: i32,
+	) -> Result<()> {
+		Ok(())
+	}
+
+	/// https://wayland.app/protocols/wayland#wl_surface:request:offset
+	#[tracing::instrument(level = "debug", skip_all)]
+	async fn offset(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_x: i32,
+		_y: i32,
+	) -> Result<()> {
+		Ok(())
+	}
+
+	/// https://wayland.app/protocols/wayland#wl_surface:request:destroy
+	#[tracing::instrument(level = "debug", skip_all)]
+	async fn destroy(&self, _client: &mut Client, _sender_id: ObjectId) -> Result<()> {
+		Ok(())
+	}
+}
+
+impl Drop for Surface {
+	fn drop(&mut self) {
+		self.role.lock().take();
+	}
+}

src/wayland/core/touch.rs

@@ -0,0 +1,75 @@
+use crate::wayland::core::surface::Surface;
+use mint::Vector2;
+use std::sync::Arc;
+pub use waynest::server::protocol::core::wayland::wl_touch::*;
+use waynest::{
+	server::{Client, Dispatcher, Result},
+	wire::ObjectId,
+};
+
+use super::seat::fixed_from_f32;
+
+#[derive(Debug, Dispatcher)]
+pub struct Touch(pub ObjectId);
+impl Touch {
+	pub async fn handle_touch_down(
+		&self,
+		client: &mut Client,
+		surface: Arc<Surface>,
+		id: u32,
+		position: Vector2<f32>,
+	) -> Result<()> {
+		let serial = client.next_event_serial();
+		self.down(
+			client,
+			self.0,
+			serial,
+			0,
+			surface.id,
+			id as i32,
+			fixed_from_f32(position.x),
+			fixed_from_f32(position.y),
+		)
+		.await?;
+		self.frame(client, self.0).await
+	}
+
+	pub async fn handle_touch_move(
+		&self,
+		client: &mut Client,
+		id: u32,
+		position: Vector2<f32>,
+	) -> Result<()> {
+		self.motion(
+			client,
+			self.0,
+			0,
+			id as i32,
+			fixed_from_f32(position.x),
+			fixed_from_f32(position.y),
+		)
+		.await?;
+		self.frame(client, self.0).await
+	}
+
+	pub async fn handle_touch_up(&self, client: &mut Client, id: u32) -> Result<()> {
+		let serial = client.next_event_serial();
+		self.up(client, self.0, serial, 0, id as i32).await?;
+		self.frame(client, self.0).await
+	}
+
+	pub async fn reset(&self, client: &mut Client) -> Result<()> {
+		self.frame(client, self.0).await
+	}
+}
+
+impl WlTouch for Touch {
+	/// https://wayland.app/protocols/wayland#wl_touch:request:release
+	async fn release(
+		&self,
+		_client: &mut waynest::server::Client,
+		_sender_id: waynest::wire::ObjectId,
+	) -> Result<()> {
+		Ok(())
+	}
+}

src/wayland/data_device.rs

@@ -1,100 +0,0 @@
-use smithay::reexports::wayland_server::{
-	protocol::{
-		wl_data_device::{
-			Request::{Release, SetSelection, StartDrag},
-			WlDataDevice,
-		},
-		wl_data_device_manager::{
-			Request::{CreateDataSource, GetDataDevice},
-			WlDataDeviceManager,
-		},
-		wl_data_source::{
-			Request::{Destroy, Offer, SetActions},
-			WlDataSource,
-		},
-	},
-	Client, DataInit, Dispatch, DisplayHandle, GlobalDispatch, New, Resource,
-};
-
-use super::state::WaylandState;
-
-impl GlobalDispatch<WlDataDeviceManager, (), WaylandState> for WaylandState {
-	fn bind(
-		_state: &mut WaylandState,
-		_handle: &DisplayHandle,
-		_client: &Client,
-		resource: New<WlDataDeviceManager>,
-		_global_data: &(),
-		data_init: &mut DataInit<'_, WaylandState>,
-	) {
-		let _resource = data_init.init(resource, ());
-	}
-}
-
-impl Dispatch<WlDataDeviceManager, (), WaylandState> for WaylandState {
-	fn request(
-		_state: &mut WaylandState,
-		_client: &Client,
-		_resource: &WlDataDeviceManager,
-		request: <WlDataDeviceManager as Resource>::Request,
-		_data: &(),
-		_dhandle: &DisplayHandle,
-		data_init: &mut DataInit<'_, WaylandState>,
-	) {
-		match request {
-			CreateDataSource { id } => {
-				data_init.init(id, ());
-			}
-			GetDataDevice { id, seat: _ } => {
-				data_init.init(id, ());
-			}
-			_ => unreachable!(),
-		}
-	}
-}
-
-impl Dispatch<WlDataSource, (), WaylandState> for WaylandState {
-	fn request(
-		_state: &mut WaylandState,
-		_client: &Client,
-		_resource: &WlDataSource,
-		request: <WlDataSource as Resource>::Request,
-		_data: &(),
-		_dhandle: &DisplayHandle,
-		_data_init: &mut DataInit<'_, WaylandState>,
-	) {
-		match request {
-			Offer { mime_type: _ } => {}
-			Destroy => {}
-			SetActions { dnd_actions: _ } => {}
-			_ => unreachable!(),
-		}
-	}
-}
-
-impl Dispatch<WlDataDevice, (), WaylandState> for WaylandState {
-	fn request(
-		_state: &mut WaylandState,
-		_client: &Client,
-		_resource: &WlDataDevice,
-		request: <WlDataDevice as Resource>::Request,
-		_data: &(),
-		_dhandle: &DisplayHandle,
-		_data_init: &mut DataInit<'_, WaylandState>,
-	) {
-		match request {
-			StartDrag {
-				source: _,
-				origin: _,
-				icon: _,
-				serial: _,
-			} => {}
-			SetSelection {
-				source: _,
-				serial: _,
-			} => {}
-			Release => {}
-			_ => unreachable!(),
-		}
-	}
-}

src/wayland/decoration.rs

@@ -1,97 +0,0 @@
-use super::state::WaylandState;
-use smithay::{
-	delegate_kde_decoration,
-	reexports::{
-		wayland_protocols::xdg::{
-			decoration::zv1::server::{
-				zxdg_decoration_manager_v1::{self, ZxdgDecorationManagerV1},
-				zxdg_toplevel_decoration_v1::{self, Mode, ZxdgToplevelDecorationV1},
-			},
-			shell::server::xdg_toplevel::XdgToplevel,
-		},
-		wayland_protocols_misc::server_decoration::server::org_kde_kwin_server_decoration::{
-			Mode as KdeMode, OrgKdeKwinServerDecoration,
-		},
-		wayland_server::{
-			protocol::wl_surface::WlSurface, Client, DataInit, Dispatch, DisplayHandle,
-			GlobalDispatch, New, Resource, WEnum, Weak,
-		},
-	},
-	wayland::shell::{self, kde::decoration::KdeDecorationHandler},
-};
-
-impl GlobalDispatch<ZxdgDecorationManagerV1, (), WaylandState> for WaylandState {
-	fn bind(
-		_state: &mut WaylandState,
-		_handle: &DisplayHandle,
-		_client: &Client,
-		resource: New<ZxdgDecorationManagerV1>,
-		_global_data: &(),
-		data_init: &mut DataInit<'_, WaylandState>,
-	) {
-		data_init.init(resource, ());
-	}
-}
-
-impl Dispatch<ZxdgDecorationManagerV1, (), WaylandState> for WaylandState {
-	fn request(
-		_state: &mut WaylandState,
-		_client: &Client,
-		_resource: &ZxdgDecorationManagerV1,
-		request: zxdg_decoration_manager_v1::Request,
-		_data: &(),
-		_dhandle: &DisplayHandle,
-		data_init: &mut DataInit<'_, WaylandState>,
-	) {
-		match request {
-			zxdg_decoration_manager_v1::Request::Destroy => (),
-			zxdg_decoration_manager_v1::Request::GetToplevelDecoration { id, toplevel } => {
-				data_init.init(id, toplevel.downgrade());
-			}
-			_ => unreachable!(),
-		}
-	}
-}
-impl Dispatch<ZxdgToplevelDecorationV1, Weak<XdgToplevel>, WaylandState> for WaylandState {
-	fn request(
-		_state: &mut WaylandState,
-		_client: &Client,
-		resource: &ZxdgToplevelDecorationV1,
-		request: zxdg_toplevel_decoration_v1::Request,
-		_data: &Weak<XdgToplevel>,
-		_dhandle: &DisplayHandle,
-		_data_init: &mut DataInit<'_, WaylandState>,
-	) {
-		match request {
-			zxdg_toplevel_decoration_v1::Request::SetMode { mode: _ } => {
-				resource.configure(Mode::ServerSide);
-			}
-			zxdg_toplevel_decoration_v1::Request::UnsetMode => {
-				resource.configure(Mode::ServerSide);
-			}
-			zxdg_toplevel_decoration_v1::Request::Destroy => (),
-			_ => unreachable!(),
-		}
-	}
-}
-
-impl KdeDecorationHandler for WaylandState {
-	fn kde_decoration_state(&self) -> &shell::kde::decoration::KdeDecorationState {
-		&self.kde_decoration_state
-	}
-
-	fn new_decoration(&mut self, _surface: &WlSurface, decoration: &OrgKdeKwinServerDecoration) {
-		decoration.mode(KdeMode::Server);
-	}
-
-	fn request_mode(
-		&mut self,
-		_surface: &WlSurface,
-		decoration: &OrgKdeKwinServerDecoration,
-		mode: WEnum<KdeMode>,
-	) {
-		let Ok(mode) = mode.into_result() else { return };
-		decoration.mode(mode);
-	}
-}
-delegate_kde_decoration!(WaylandState);

src/wayland/display.rs

@@ -0,0 +1,78 @@
+#![allow(unused)]
+
+use crate::wayland::{
+	MessageSink,
+	core::{
+		callback::{Callback, WlCallback},
+		output::Output,
+		seat::Seat,
+	},
+	registry::Registry,
+};
+use global_counter::primitive::exact::CounterU32;
+use std::{
+	sync::{Arc, OnceLock},
+	time::Instant,
+};
+pub use waynest::server::protocol::core::wayland::wl_display::*;
+use waynest::{
+	server::{Client, Dispatcher, Result},
+	wire::ObjectId,
+};
+
+#[derive(Dispatcher)]
+pub struct Display {
+	pub message_sink: MessageSink,
+	pub pid: Option<i32>,
+	pub seat: OnceLock<Arc<Seat>>,
+	pub output: OnceLock<Arc<Output>>,
+	id_counter: CounterU32,
+	pub creation_time: Instant,
+}
+impl Display {
+	pub fn new(message_sink: MessageSink, pid: Option<i32>) -> Self {
+		Self {
+			message_sink,
+			pid,
+			seat: OnceLock::new(),
+			output: OnceLock::new(),
+			id_counter: CounterU32::new(0xff000000), // Start at 0xff000000 to avoid conflicts with client-generated IDs
+			creation_time: Instant::now(),
+		}
+	}
+	pub fn next_server_id(&self) -> ObjectId {
+		unsafe { ObjectId::from_raw(self.id_counter.inc()) }
+	}
+}
+impl WlDisplay for Display {
+	/// https://wayland.app/protocols/wayland#wl_display:request:sync
+	async fn sync(
+		&self,
+		client: &mut Client,
+		sender_id: ObjectId,
+		callback_id: ObjectId,
+	) -> Result<()> {
+		let serial = client.next_event_serial();
+		Callback(callback_id)
+			.done(client, callback_id, serial)
+			.await?;
+
+		self.delete_id(client, sender_id, callback_id.as_raw())
+			.await?;
+		Ok(())
+	}
+
+	/// https://wayland.app/protocols/wayland#wl_display:request:get_registry
+	async fn get_registry(
+		&self,
+		client: &mut Client,
+		_sender_id: ObjectId,
+		registry_id: ObjectId,
+	) -> Result<()> {
+		let registry = client.insert(registry_id, Registry);
+
+		registry.advertise_globals(client, registry_id).await?;
+
+		Ok(())
+	}
+}

src/wayland/dmabuf/buffer_backing.rs

@@ -0,0 +1,120 @@
+use super::buffer_params::BufferParams;
+use crate::wayland::RENDER_DEVICE;
+use bevy::{
+	asset::{Assets, Handle},
+	image::Image,
+};
+use bevy_dmabuf::{
+	dmatex::{Dmatex, Resolution},
+	import::{DropCallback, ImportError, ImportedDmatexs, ImportedTexture, import_texture},
+};
+use drm_fourcc::DrmFourcc;
+use mint::Vector2;
+use parking_lot::Mutex;
+use std::sync::{Arc, OnceLock};
+use tracing::info;
+use waynest::server::protocol::stable::linux_dmabuf_v1::zwp_linux_buffer_params_v1::Flags;
+
+/// Parameters for a shared memory buffer
+pub struct DmabufBacking {
+	size: Vector2<u32>,
+	format: DrmFourcc,
+	tex: OnceLock<Handle<Image>>,
+	pending_imported_dmatex: Mutex<Option<ImportedTexture>>,
+}
+
+impl std::fmt::Debug for DmabufBacking {
+	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+		f.debug_struct("DmabufBacking")
+			.field("size", &self.size)
+			.field("format", &self.format)
+			.field("tex", &self.tex)
+			.finish()
+	}
+}
+
+impl DmabufBacking {
+	#[tracing::instrument(level = "debug", skip_all)]
+	pub fn new(dmatex: Dmatex) -> Result<Self, ImportError> {
+		let dev = RENDER_DEVICE.wait();
+
+		Ok(Self {
+			size: [dmatex.res.x, dmatex.res.y].into(),
+			format: DrmFourcc::try_from(dmatex.format).unwrap(),
+			tex: OnceLock::new(),
+			pending_imported_dmatex: Mutex::new(Some(import_texture(
+				dev,
+				dmatex,
+				DropCallback(None),
+			)?)),
+		})
+	}
+
+	#[tracing::instrument(level = "debug", skip_all)]
+	pub fn from_params(
+		params: Arc<BufferParams>,
+		size: Vector2<u32>,
+		format: DrmFourcc,
+		flags: Flags,
+	) -> Result<Self, ImportError> {
+		tracing::info!("Creating new DmabufBacking");
+		let mut planes = Vec::from_iter(std::mem::take(&mut *params.planes.lock()));
+		planes.sort_by_key(|(index, _)| *index);
+		let planes = planes.into_iter().map(|(_, tex)| tex).collect::<Vec<_>>();
+		let dmatex = Dmatex {
+			planes,
+			res: Resolution {
+				x: size.x,
+				y: size.y,
+			},
+			format: format as u32,
+			// TODO: impl this in bevy-dmabuf
+			flip_y: flags.contains(Flags::YInvert),
+			srgb: true,
+		};
+
+		DmabufBacking::new(dmatex)
+	}
+
+	#[tracing::instrument(level = "debug", skip_all)]
+	pub fn update_tex(
+		&self,
+		dmatexes: &ImportedDmatexs,
+		images: &mut Assets<Image>,
+	) -> Option<Handle<Image>> {
+		info!("updating dmabuf tex");
+		self.pending_imported_dmatex
+			.lock()
+			.take()
+			.map(|tex| dmatexes.insert_imported_dmatex(images, tex))
+			.inspect(|handle| {
+				_ = self.tex.set(handle.clone());
+			})
+	}
+
+	pub fn is_transparent(&self) -> bool {
+		matches!(
+			self.format,
+			DrmFourcc::Abgr1555
+				| DrmFourcc::Abgr16161616f
+				| DrmFourcc::Abgr2101010
+				| DrmFourcc::Abgr4444
+				| DrmFourcc::Abgr8888
+				| DrmFourcc::Argb1555
+				| DrmFourcc::Argb16161616f
+				| DrmFourcc::Argb2101010
+				| DrmFourcc::Argb4444
+				| DrmFourcc::Argb8888
+				| DrmFourcc::Axbxgxrx106106106106
+				| DrmFourcc::Ayuv
+				| DrmFourcc::Rgba1010102
+				| DrmFourcc::Rgba4444
+				| DrmFourcc::Rgba5551
+				| DrmFourcc::Rgba8888
+		)
+	}
+
+	pub fn size(&self) -> Vector2<usize> {
+		[self.size.x as usize, self.size.y as usize].into()
+	}
+}

src/wayland/dmabuf/buffer_params.rs

@@ -0,0 +1,180 @@
+use super::buffer_backing::DmabufBacking;
+use crate::wayland::{
+	core::buffer::{Buffer, BufferBacking},
+	util::ClientExt,
+};
+use bevy_dmabuf::dmatex::DmatexPlane;
+use drm_fourcc::DrmFourcc;
+use parking_lot::Mutex;
+use rustc_hash::FxHashMap;
+use std::os::fd::{AsRawFd, OwnedFd};
+use waynest::{
+	server::{
+		Client, Dispatcher, Result,
+		protocol::stable::linux_dmabuf_v1::zwp_linux_buffer_params_v1::{
+			Error, Flags, ZwpLinuxBufferParamsV1,
+		},
+	},
+	wire::ObjectId,
+};
+
+/// Parameters for creating a DMA-BUF-based wl_buffer
+///
+/// This is a temporary object that collects dmabufs and other parameters
+/// that together form a single logical buffer. The object may eventually
+/// create one wl_buffer unless cancelled by destroying it.
+#[derive(Debug, Dispatcher)]
+pub struct BufferParams {
+	pub id: ObjectId,
+	pub(super) planes: Mutex<FxHashMap<u32, DmatexPlane>>,
+}
+
+impl BufferParams {
+	#[tracing::instrument(level = "debug", skip_all)]
+	pub fn new(id: ObjectId) -> Self {
+		tracing::info!("Creating new BufferParams with id {:?}", id);
+		Self {
+			id,
+			planes: Mutex::new(FxHashMap::default()),
+		}
+	}
+}
+
+impl ZwpLinuxBufferParamsV1 for BufferParams {
+	async fn destroy(&self, _client: &mut Client, _sender_id: ObjectId) -> Result<()> {
+		tracing::info!("Destroying BufferParams {:?}", self.id);
+		Ok(())
+	}
+
+	#[tracing::instrument(level = "debug", skip_all)]
+	async fn add(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		fd: OwnedFd,
+		plane_idx: u32,
+		offset: u32,
+		stride: u32,
+		modifier_hi: u32,
+		modifier_lo: u32,
+	) -> Result<()> {
+		let fd_num = fd.as_raw_fd();
+		tracing::info!(
+			"Adding plane {} with fd {} to BufferParams {:?}",
+			plane_idx,
+			fd_num,
+			self.id
+		);
+
+		let mut planes = self.planes.lock();
+
+		// Check if plane index is already set
+		if planes.contains_key(&plane_idx) {
+			tracing::error!(
+				"Plane {} already exists in BufferParams {:?}",
+				plane_idx,
+				self.id
+			);
+			return Err(waynest::server::Error::MissingObject(self.id));
+		}
+
+		// Create plane with the provided parameters
+		let plane = DmatexPlane {
+			dmabuf_fd: fd.into(),
+			offset,
+			stride: stride as i32,
+			modifier: ((modifier_hi as u64) << 32) | (modifier_lo as u64),
+		};
+
+		// Store the plane
+		planes.insert(plane_idx, plane);
+		Ok(())
+	}
+
+	#[tracing::instrument(level = "debug", skip_all)]
+	async fn create(
+		&self,
+		client: &mut Client,
+		_sender_id: ObjectId,
+		width: i32,
+		height: i32,
+		format: u32,
+		flags: Flags,
+	) -> Result<()> {
+		tracing::info!("Creating buffer from BufferParams {:?}", self.id);
+		// Create the buffer with DMA-BUF backing using self as the backing
+		let size = [width as u32, height as u32].into();
+		let buffer = DmabufBacking::from_params(
+			client.get::<Self>(self.id).unwrap(),
+			size,
+			DrmFourcc::try_from(format).unwrap(),
+			flags,
+		)
+		.inspect_err(|e| tracing::error!("Failed to import dmabuf because {e}"))
+		.map(|backing| {
+			let id = client.display().next_server_id();
+			Buffer::new(client, id, BufferBacking::Dmabuf(backing))
+		});
+
+		match buffer {
+			Ok(buffer) => self.created(client, self.id, buffer.id).await,
+			Err(_) => {
+				client.remove(self.id);
+				self.failed(client, self.id).await
+			}
+		}
+	}
+
+	#[tracing::instrument(level = "debug", skip_all)]
+	async fn create_immed(
+		&self,
+		client: &mut Client,
+		sender_id: ObjectId,
+		buffer_id: ObjectId,
+		width: i32,
+		height: i32,
+		format: u32,
+		flags: Flags,
+	) -> Result<()> {
+		// TODO: terminate client on fail, or send a fail event or something
+		// Create the buffer with DMA-BUF backing using self as the backing
+		match DmabufBacking::from_params(
+			client.get::<Self>(self.id).unwrap(),
+			[width as u32, height as u32].into(),
+			DrmFourcc::try_from(format).unwrap(),
+			flags,
+		) {
+			Ok(backing) => {
+				Buffer::new(client, buffer_id, BufferBacking::Dmabuf(backing));
+			}
+			Err(e) => {
+				client
+					.protocol_error(
+						sender_id,
+						buffer_id,
+						Error::Incomplete as u32,
+						format!("Failed to import dmabuf because {e}"),
+					)
+					.await?;
+				tracing::error!("Failed to import dmabuf because {e}");
+			}
+		}
+		Ok(())
+	}
+}
+
+impl Drop for BufferParams {
+	#[tracing::instrument(level = "debug", skip_all)]
+	fn drop(&mut self) {
+		let planes = self.planes.get_mut();
+		tracing::info!("BufferParams being dropped with {} planes", planes.len());
+		for (idx, plane) in planes.iter() {
+			tracing::info!(
+				"Dropping plane {} with fd {}",
+				idx,
+				plane.dmabuf_fd.as_raw_fd()
+			);
+		}
+		planes.clear();
+	}
+}

src/wayland/dmabuf/feedback.rs

@@ -0,0 +1,102 @@
+use super::Dmabuf;
+use crate::wayland::vulkano_data::VULKANO_CONTEXT;
+use memfd::MemfdOptions;
+use std::{
+	io::Write,
+	os::fd::{FromRawFd, IntoRawFd, OwnedFd},
+	sync::Arc,
+};
+use waynest::{
+	server::{
+		Client, Dispatcher, Result,
+		protocol::stable::linux_dmabuf_v1::zwp_linux_dmabuf_feedback_v1::{
+			TrancheFlags, ZwpLinuxDmabufFeedbackV1,
+		},
+	},
+	wire::ObjectId,
+};
+
+#[derive(Debug, Dispatcher)]
+pub struct DmabufFeedback(pub Arc<Dmabuf>);
+impl DmabufFeedback {
+	#[tracing::instrument(level = "debug", skip_all)]
+	pub async fn send_params(&self, client: &mut Client, sender_id: ObjectId) -> Result<()> {
+		let num_formats = self.0.formats.len();
+		// Send format table first
+		self.send_format_table(client, sender_id).await?;
+
+		// Get the device information from Vulkan properties
+		let props = VULKANO_CONTEXT.get().unwrap().phys_dev.properties();
+
+		// Create dev_t from the primary node major/minor numbers
+		let primary_dev_id = {
+			let major = props.primary_major.unwrap() as u64;
+			let minor = props.primary_minor.unwrap() as u64;
+			// On Linux, dev_t is created with makedev(major, minor)
+			// which is ((major & 0xfffff000) << 32) | ((major & 0xfff) << 8) | (minor & 0xff)
+			((major & 0xfffff000) << 32) | ((major & 0xfff) << 8) | (minor & 0xff)
+		};
+		let dev_id = primary_dev_id.to_ne_bytes().to_vec();
+
+		// Send main device
+		self.main_device(client, sender_id, dev_id.clone()).await?;
+
+		// Send tranche with same device since we only support the main GPU
+		self.tranche_target_device(client, sender_id, dev_id)
+			.await?;
+
+		let indices = (0..num_formats)
+			.flat_map(|i| (i as u16).to_ne_bytes())
+			.collect();
+		self.tranche_formats(client, sender_id, indices).await?;
+
+		// No special flags needed for simple EGL texture usage
+		self.tranche_flags(client, sender_id, TrancheFlags::empty())
+			.await?;
+
+		// Mark tranche complete
+		self.tranche_done(client, sender_id).await?;
+
+		// Mark overall feedback complete
+		self.done(client, sender_id).await?;
+		Ok(())
+	}
+
+	#[tracing::instrument(level = "debug", skip_all)]
+	pub async fn send_format_table(&self, client: &mut Client, sender_id: ObjectId) -> Result<()> {
+		// Format + modifier pair (16 bytes):
+		// - format: u32
+		// - padding: 4 bytes
+		// - modifier: u64
+		let size = self.0.formats.len() as u32 * 16u32;
+		// Create a temporary file for the format table
+		let mfd = MemfdOptions::default()
+			.create("stardustxr-format-table")
+			.map_err(|e| waynest::server::Error::Custom(e.to_string()))?;
+
+		mfd.as_file().set_len(size as u64)?;
+
+		for (format, modifier) in self.0.formats.iter() {
+			let format = *format as u32;
+			// Write the format+modifier pair
+			mfd.as_file().write_all(&format.to_ne_bytes())?;
+			mfd.as_file().write_all(&0_u32.to_ne_bytes())?;
+			mfd.as_file().write_all(&modifier.to_ne_bytes())?;
+		}
+
+		self.format_table(
+			client,
+			sender_id,
+			unsafe { OwnedFd::from_raw_fd(mfd.into_raw_fd()) },
+			size,
+		)
+		.await?;
+		Ok(())
+	}
+}
+
+impl ZwpLinuxDmabufFeedbackV1 for DmabufFeedback {
+	async fn destroy(&self, _client: &mut Client, _sender_id: ObjectId) -> Result<()> {
+		Ok(())
+	}
+}

src/wayland/dmabuf/mod.rs

@@ -0,0 +1,281 @@
+pub mod buffer_backing;
+pub mod buffer_params;
+pub mod feedback;
+
+use std::sync::LazyLock;
+
+use super::{util::ClientExt, vulkano_data::VULKANO_CONTEXT};
+use crate::core::registry::Registry;
+use bevy_dmabuf::{format_mapping::drm_fourcc_to_vk_format, wgpu_init::vulkan_to_wgpu};
+use buffer_params::BufferParams;
+use drm_fourcc::DrmFourcc;
+use feedback::DmabufFeedback;
+use rustc_hash::FxHashSet;
+use vulkano::format::FormatFeatures;
+use waynest::{
+	server::{
+		Client, Dispatcher, Error, Result,
+		protocol::stable::linux_dmabuf_v1::zwp_linux_dmabuf_v1::ZwpLinuxDmabufV1,
+	},
+	wire::ObjectId,
+};
+
+pub static DMABUF_FORMATS: LazyLock<Vec<(DrmFourcc, u64)>> = LazyLock::new(|| {
+	let vk = VULKANO_CONTEXT.wait();
+
+	let format_modifier_pairs = ALL_DRM_FOURCCS
+		.iter()
+		.copied()
+		.filter_map(|f| Some((f, drm_fourcc_to_vk_format(f)?)))
+		.filter(|(_, vk_format)| vulkan_to_wgpu(*vk_format).is_some())
+		.filter_map(|(f, vk_format)| {
+			Some((
+				f,
+				vk.phys_dev
+					.format_properties(vk_format.try_into().unwrap())
+					.ok()?
+					.drm_format_modifier_properties
+					.into_iter()
+					.filter(|v| {
+						v.drm_format_modifier_tiling_features
+							.contains(FormatFeatures::SAMPLED_IMAGE)
+					})
+					.map(|v| v.drm_format_modifier)
+					.collect::<Vec<_>>(),
+			))
+		})
+		.flat_map(|(f, mods)| mods.into_iter().map(move |modifier| (f, modifier)))
+		.collect::<FxHashSet<_>>();
+
+	let mut format_modifier_pairs = format_modifier_pairs.into_iter().collect::<Vec<_>>();
+	format_modifier_pairs.sort_by(|(f1, m1), (f2, m2)| {
+		// Prioritize LINEAR modifier
+		let linear1 = *m1 == 0;
+		let linear2 = *m2 == 0;
+		linear2
+			.cmp(&linear1) // true = 1, false = 0
+			.then_with(|| (*f1 as u32).cmp(&(*f2 as u32))) // Sort by format numerically
+			.then_with(|| m1.cmp(m2)) // Then by modifier
+	});
+	format_modifier_pairs
+});
+
+/// Main DMA-BUF interface implementation
+///
+/// This interface allows clients to create wl_buffers from DMA-BUFs.
+/// It handles:
+/// - Format/modifier advertisement
+/// - Buffer parameter creation
+/// - Default/surface-specific feedback
+///
+/// The implementation ensures:
+/// - Coherency for read access in dmabuf data
+/// - Proper lifetime management of dmabuf file descriptors
+/// - Safe handling of buffer attachments
+#[derive(Debug, Dispatcher)]
+pub struct Dmabuf {
+	// Track supported formats and modifiers
+	// formats: Mutex<FxHashSet<DrmFormat>>,
+	// Track active buffer parameters objects by their ID
+	active_params: Registry<BufferParams>,
+	pub(self) version: u32,
+	pub(self) formats: Vec<(DrmFourcc, u64)>,
+}
+
+impl Dmabuf {
+	/// Create a new DMA-BUF interface instance
+	pub async fn new(client: &mut Client, id: ObjectId, version: u32) -> Result<Self> {
+		let dmabuf = Self {
+			active_params: Registry::new(),
+			version,
+			formats: DMABUF_FORMATS.clone(),
+		};
+
+		if version < 3 {
+			for (format, _) in &dmabuf.formats {
+				dmabuf.format(client, id, *format as u32).await?;
+			}
+		}
+		// `modifier` is deprecated in version 4
+		if version == 3 {
+			for (format, modifier) in &dmabuf.formats {
+				let format = *format as u32;
+				let modifier_hi = (*modifier >> 32) as u32;
+				let modifier_lo = *modifier as u32;
+				dmabuf
+					.modifier(client, id, format, modifier_hi, modifier_lo)
+					.await?;
+			}
+		}
+
+		Ok(dmabuf)
+	}
+
+	/// Remove a buffer parameters object from tracking
+	pub(crate) fn remove_params(&self, params_id: ObjectId) {
+		self.active_params.retain(|params| params.id != params_id);
+	}
+}
+
+impl ZwpLinuxDmabufV1 for Dmabuf {
+	async fn destroy(&self, _client: &mut Client, sender_id: ObjectId) -> Result<()> {
+		self.remove_params(sender_id);
+		Ok(())
+	}
+
+	async fn create_params(
+		&self,
+		client: &mut Client,
+		_sender_id: ObjectId,
+		params_id: ObjectId,
+	) -> Result<()> {
+		// Create new buffer parameters object
+		let params = client.insert(params_id, BufferParams::new(params_id));
+		self.active_params.add_raw(&params);
+		Ok(())
+	}
+
+	async fn get_default_feedback(
+		&self,
+		client: &mut Client,
+		sender_id: ObjectId,
+		id: ObjectId,
+	) -> Result<()> {
+		if self.version < 3 {
+			client
+				.protocol_error(
+					sender_id,
+					id,
+					71,
+					"Can't call get_default_feedback on version < 4 of dmabuf".into(),
+				)
+				.await?;
+			return Err(Error::Custom("Protocol error".into()));
+		}
+		// Create feedback object for default (non-surface-specific) settings
+		let feedback = client.insert(id, DmabufFeedback(client.get::<Dmabuf>(sender_id).unwrap()));
+		feedback.send_params(client, id).await?;
+		Ok(())
+	}
+
+	async fn get_surface_feedback(
+		&self,
+		client: &mut Client,
+		sender_id: ObjectId,
+		id: ObjectId,
+		_surface: ObjectId,
+	) -> Result<()> {
+		// Create feedback object for surface-specific settings
+		// Note: Surface-specific feedback could be optimized based on the surface's
+		// requirements, but for now we use the same feedback as default
+		self.get_default_feedback(client, sender_id, id).await
+	}
+}
+
+pub const ALL_DRM_FOURCCS: [DrmFourcc; 105] = [
+	DrmFourcc::Abgr1555,
+	DrmFourcc::Abgr16161616f,
+	DrmFourcc::Abgr2101010,
+	DrmFourcc::Abgr4444,
+	DrmFourcc::Abgr8888,
+	DrmFourcc::Argb1555,
+	DrmFourcc::Argb16161616f,
+	DrmFourcc::Argb2101010,
+	DrmFourcc::Argb4444,
+	DrmFourcc::Argb8888,
+	DrmFourcc::Axbxgxrx106106106106,
+	DrmFourcc::Ayuv,
+	DrmFourcc::Bgr233,
+	DrmFourcc::Bgr565,
+	DrmFourcc::Bgr565_a8,
+	DrmFourcc::Bgr888,
+	DrmFourcc::Bgr888_a8,
+	DrmFourcc::Bgra1010102,
+	DrmFourcc::Bgra4444,
+	DrmFourcc::Bgra5551,
+	DrmFourcc::Bgra8888,
+	DrmFourcc::Bgrx1010102,
+	DrmFourcc::Bgrx4444,
+	DrmFourcc::Bgrx5551,
+	DrmFourcc::Bgrx8888,
+	DrmFourcc::Bgrx8888_a8,
+	DrmFourcc::Big_endian,
+	DrmFourcc::C8,
+	DrmFourcc::Gr1616,
+	DrmFourcc::Gr88,
+	DrmFourcc::Nv12,
+	DrmFourcc::Nv15,
+	DrmFourcc::Nv16,
+	DrmFourcc::Nv21,
+	DrmFourcc::Nv24,
+	DrmFourcc::Nv42,
+	DrmFourcc::Nv61,
+	DrmFourcc::P010,
+	DrmFourcc::P012,
+	DrmFourcc::P016,
+	DrmFourcc::P210,
+	DrmFourcc::Q401,
+	DrmFourcc::Q410,
+	DrmFourcc::R16,
+	DrmFourcc::R8,
+	DrmFourcc::Rg1616,
+	DrmFourcc::Rg88,
+	DrmFourcc::Rgb332,
+	DrmFourcc::Rgb565,
+	DrmFourcc::Rgb565_a8,
+	DrmFourcc::Rgb888,
+	DrmFourcc::Rgb888_a8,
+	DrmFourcc::Rgba1010102,
+	DrmFourcc::Rgba4444,
+	DrmFourcc::Rgba5551,
+	DrmFourcc::Rgba8888,
+	DrmFourcc::Rgbx1010102,
+	DrmFourcc::Rgbx4444,
+	DrmFourcc::Rgbx5551,
+	DrmFourcc::Rgbx8888,
+	DrmFourcc::Rgbx8888_a8,
+	DrmFourcc::Uyvy,
+	DrmFourcc::Vuy101010,
+	DrmFourcc::Vuy888,
+	DrmFourcc::Vyuy,
+	DrmFourcc::X0l0,
+	DrmFourcc::X0l2,
+	DrmFourcc::Xbgr1555,
+	DrmFourcc::Xbgr16161616f,
+	DrmFourcc::Xbgr2101010,
+	DrmFourcc::Xbgr4444,
+	DrmFourcc::Xbgr8888,
+	DrmFourcc::Xbgr8888_a8,
+	DrmFourcc::Xrgb1555,
+	DrmFourcc::Xrgb16161616f,
+	DrmFourcc::Xrgb2101010,
+	DrmFourcc::Xrgb4444,
+	DrmFourcc::Xrgb8888,
+	DrmFourcc::Xrgb8888_a8,
+	DrmFourcc::Xvyu12_16161616,
+	DrmFourcc::Xvyu16161616,
+	DrmFourcc::Xvyu2101010,
+	DrmFourcc::Xyuv8888,
+	DrmFourcc::Y0l0,
+	DrmFourcc::Y0l2,
+	DrmFourcc::Y210,
+	DrmFourcc::Y212,
+	DrmFourcc::Y216,
+	DrmFourcc::Y410,
+	DrmFourcc::Y412,
+	DrmFourcc::Y416,
+	DrmFourcc::Yuv410,
+	DrmFourcc::Yuv411,
+	DrmFourcc::Yuv420,
+	DrmFourcc::Yuv420_10bit,
+	DrmFourcc::Yuv420_8bit,
+	DrmFourcc::Yuv422,
+	DrmFourcc::Yuv444,
+	DrmFourcc::Yuyv,
+	DrmFourcc::Yvu410,
+	DrmFourcc::Yvu411,
+	DrmFourcc::Yvu420,
+	DrmFourcc::Yvu422,
+	DrmFourcc::Yvu444,
+	DrmFourcc::Yvyu,
+];

src/wayland/drm.rs

@@ -1,144 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-only
-
-// Re-export only the actual code, and then only use this re-export
-// The `generated` module below is just some boilerplate to properly isolate stuff
-// and avoid exposing internal details.
-//
-// You can use all the types from my_protocol as if they went from `wayland_client::protocol`.
-pub use generated::wl_drm;
-
-#[allow(non_upper_case_globals, non_camel_case_types)]
-mod generated {
-	use smithay::reexports::wayland_server::{self, protocol::*};
-
-	pub mod __interfaces {
-		use smithay::reexports::wayland_server::protocol::__interfaces::*;
-		wayland_scanner::generate_interfaces!("src/wayland/wayland-drm.xml");
-	}
-	use self::__interfaces::*;
-
-	wayland_scanner::generate_server_code!("src/wayland/wayland-drm.xml");
-}
-
-use super::state::WaylandState;
-use smithay::{
-	backend::allocator::{
-		dmabuf::{Dmabuf, DmabufFlags},
-		Fourcc, Modifier,
-	},
-	reexports::wayland_server::{
-		Client, DataInit, Dispatch, DisplayHandle, GlobalDispatch, New, Resource,
-	},
-};
-use std::convert::TryFrom;
-
-impl GlobalDispatch<wl_drm::WlDrm, (), WaylandState> for WaylandState {
-	fn bind(
-		state: &mut WaylandState,
-		_dh: &DisplayHandle,
-		_client: &Client,
-		resource: New<wl_drm::WlDrm>,
-		_global_data: &(),
-		data_init: &mut DataInit<'_, WaylandState>,
-	) {
-		let drm_instance = data_init.init(resource, ());
-
-		drm_instance.device("/dev/dri/renderD128".to_string());
-		if drm_instance.version() >= 2 {
-			drm_instance.capabilities(wl_drm::Capability::Prime as u32);
-		}
-		for format in state.drm_formats.iter() {
-			if let Ok(converted) = wl_drm::Format::try_from(*format as u32) {
-				drm_instance.format(converted as u32);
-			}
-		}
-	}
-
-	fn can_view(_client: Client, _global_dataa: &()) -> bool {
-		true
-	}
-}
-
-impl Dispatch<wl_drm::WlDrm, (), WaylandState> for WaylandState {
-	fn request(
-		state: &mut WaylandState,
-		_client: &Client,
-		drm: &wl_drm::WlDrm,
-		request: wl_drm::Request,
-		_data: &(),
-		_dh: &DisplayHandle,
-		data_init: &mut DataInit<'_, WaylandState>,
-	) {
-		match request {
-			wl_drm::Request::Authenticate { .. } => drm.authenticated(),
-			wl_drm::Request::CreateBuffer { .. } => drm.post_error(
-				wl_drm::Error::InvalidName,
-				String::from("Flink handles are unsupported, use PRIME"),
-			),
-			wl_drm::Request::CreatePlanarBuffer { .. } => drm.post_error(
-				wl_drm::Error::InvalidName,
-				String::from("Flink handles are unsupported, use PRIME"),
-			),
-			wl_drm::Request::CreatePrimeBuffer {
-				id,
-				name,
-				width,
-				height,
-				format,
-				offset0,
-				stride0,
-				..
-			} => {
-				let format = match Fourcc::try_from(format) {
-					Ok(format) => {
-						if !state.drm_formats.contains(&format) {
-							drm.post_error(
-								wl_drm::Error::InvalidFormat,
-								String::from("Format not advertised by wl_drm"),
-							);
-							return;
-						}
-						format
-					}
-					Err(_) => {
-						drm.post_error(
-							wl_drm::Error::InvalidFormat,
-							String::from("Format unknown / not advertised by wl_drm"),
-						);
-						return;
-					}
-				};
-
-				if width < 1 || height < 1 {
-					drm.post_error(
-						wl_drm::Error::InvalidFormat,
-						String::from("width or height not positive"),
-					);
-					return;
-				}
-
-				let mut dma = Dmabuf::builder(
-					(width, height),
-					format,
-					Modifier::Invalid,
-					DmabufFlags::empty(),
-				);
-				dma.add_plane(name, 0, offset0 as u32, stride0 as u32);
-				match dma.build() {
-					Some(dmabuf) => {
-						state.dmabuf_tx.send((dmabuf.clone(), None)).unwrap();
-						data_init.init(id, dmabuf);
-					}
-					None => {
-						// Buffer import failed. The protocol documentation heavily implies killing the
-						// client is the right thing to do here.
-						drm.post_error(
-							wl_drm::Error::InvalidName,
-							"dmabuf global was destroyed on server",
-						);
-					}
-				}
-			}
-		}
-	}
-}

src/wayland/mesa_drm.rs

@@ -0,0 +1,126 @@
+use crate::wayland::{
+	core::buffer::{Buffer, BufferBacking},
+	dmabuf::{DMABUF_FORMATS, buffer_backing::DmabufBacking},
+	vulkano_data::VULKANO_CONTEXT,
+};
+use bevy_dmabuf::dmatex::{Dmatex, DmatexPlane, Resolution};
+use rustc_hash::FxHashSet;
+use std::os::fd::OwnedFd;
+use waynest::{
+	server::{Client, Dispatcher, Result, protocol::external::drm::wl_drm::*},
+	wire::ObjectId,
+};
+
+#[derive(Debug, Dispatcher, Default)]
+pub struct MesaDrm {
+	version: u32,
+}
+impl MesaDrm {
+	pub async fn new(client: &mut Client, id: ObjectId, version: u32) -> Result<MesaDrm> {
+		let drm = MesaDrm { version };
+
+		let path = {
+			// Get the device information from Vulkan properties
+			let props = VULKANO_CONTEXT.get().unwrap().phys_dev.properties();
+			let minor_version = props.render_minor.unwrap();
+			format!("/dev/dri/renderD{minor_version}")
+		};
+		drm.device(client, id, path).await?;
+
+		// this is basically just enabling ancient dmabufs lel
+		if drm.version >= 2 {
+			drm.capabilities(client, id, Capability::Prime as u32)
+				.await?;
+		}
+
+		// DRM fomrats check
+		let formats = DMABUF_FORMATS
+			.iter()
+			.map(|(fourcc, _)| fourcc)
+			.collect::<FxHashSet<_>>();
+		for format in formats {
+			drm.format(client, id, *format as u32).await?;
+		}
+
+		Ok(drm)
+	}
+}
+impl WlDrm for MesaDrm {
+	async fn authenticate(&self, client: &mut Client, sender_id: ObjectId, _id: u32) -> Result<()> {
+		self.authenticated(client, sender_id).await
+	}
+
+	async fn create_buffer(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_id: ObjectId,
+		_name: u32,
+		_width: i32,
+		_height: i32,
+		_stride: u32,
+		_format: u32,
+	) -> Result<()> {
+		tracing::error!("Tried to create non-prime wl_drm buffer!");
+		Ok(())
+	}
+
+	async fn create_planar_buffer(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_id: ObjectId,
+		_name: u32,
+		_width: i32,
+		_height: i32,
+		_format: u32,
+		_offset0: i32,
+		_stride0: i32,
+		_offset1: i32,
+		_stride1: i32,
+		_offset2: i32,
+		_stride2: i32,
+	) -> Result<()> {
+		tracing::error!("Tried to create non-prime wl_drm buffer!");
+		Ok(())
+	}
+
+	async fn create_prime_buffer(
+		&self,
+		client: &mut Client,
+		_sender_id: ObjectId,
+		buffer_id: ObjectId,
+		name: OwnedFd,
+		width: i32,
+		height: i32,
+		format: u32,
+		offset0: i32,
+		stride0: i32,
+		_offset1: i32,
+		_stride1: i32,
+		_offset2: i32,
+		_stride2: i32,
+	) -> Result<()> {
+		// TODO: actual error checking
+
+		let _ = DmabufBacking::new(Dmatex {
+			planes: vec![DmatexPlane {
+				dmabuf_fd: name.into(),
+				modifier: 72057594037927935, // because drmfourcc is so broken it doesn't actually export this, this is Invalid btw
+				offset: offset0 as u32,
+				stride: stride0,
+			}],
+			res: Resolution {
+				x: width as u32,
+				y: height as u32,
+			},
+			format,
+			flip_y: false,
+			srgb: true,
+		})
+		.inspect_err(|e| tracing::error!("Failed to import dmabuf because {e}"))
+		.map(|backing| Buffer::new(client, buffer_id, BufferBacking::Dmabuf(backing)));
+
+		Ok(())
+	}
+}

src/wayland/mod.rs

@@ -1,219 +1,390 @@
-mod compositor;
-mod data_device;
-mod decoration;
-mod seat;
-mod state;
-mod surface;
-// mod xdg_activation;
-mod drm;
-mod utils;
-mod xdg_shell;
+mod core;
+mod display;
+mod dmabuf;
+mod mesa_drm;
+mod presentation;
+mod registry;
+mod util;
+mod vulkano_data;
+mod xdg;
 
-use self::{state::WaylandState, surface::CORE_SURFACES};
-use crate::{core::task, wayland::state::ClientState};
-use color_eyre::eyre::{ensure, Result};
-use once_cell::sync::OnceCell;
-use parking_lot::Mutex;
-use smithay::backend::allocator::dmabuf::Dmabuf;
-use smithay::backend::egl::EGLContext;
-use smithay::backend::renderer::gles::GlesRenderer;
-use smithay::backend::renderer::{ImportDma, Renderer};
-use smithay::output::Output;
-use smithay::reexports::wayland_server::backend::ClientId;
-use smithay::reexports::wayland_server::DisplayHandle;
-use smithay::reexports::wayland_server::{Display, ListeningSocket};
-use smithay::wayland::dmabuf;
-use std::ffi::OsStr;
-use std::os::fd::{IntoRawFd, OwnedFd};
-use std::os::unix::prelude::AsRawFd;
+use crate::core::registry::OwnedRegistry;
+use crate::nodes::drawable::model::ModelNodeSystemSet;
+use crate::wayland::core::seat::SeatMessage;
+use crate::wayland::core::surface::Surface;
+use crate::wayland::presentation::MonotonicTimestamp;
+use crate::{
+	BevyMaterial,
+	core::{
+		error::{Result, ServerError},
+		task,
+	},
+};
+use bevy::app::{App, Plugin, Update};
+use bevy::ecs::schedule::IntoScheduleConfigs;
+use bevy::ecs::system::{Local, Res, ResMut};
+use bevy::prelude::{Deref, DerefMut};
+use bevy::render::renderer::RenderDevice;
+use bevy::render::{Render, RenderApp};
+use bevy::{asset::Assets, ecs::resource::Resource, image::Image};
+use bevy_dmabuf::import::ImportedDmatexs;
+use bevy_mod_openxr::render::end_frame;
+use bevy_mod_xr::session::XrRenderSet;
+use cluFlock::{FlockLock, ToFlock};
+use core::buffer::BufferUsage;
+use core::{buffer::Buffer, callback::Callback, surface::WL_SURFACE_REGISTRY};
+use display::Display;
+use mint::Vector2;
+use std::fs::File;
+use std::time::Duration;
 use std::{
-	ffi::c_void,
-	os::unix::{net::UnixListener, prelude::FromRawFd},
-	sync::Arc,
+	fs::{self, OpenOptions},
+	io::{self, ErrorKind},
+	os::unix::fs::OpenOptionsExt,
+	path::PathBuf,
+	sync::{Arc, OnceLock},
 };
-use stereokit_rust::system::{Backend, BackendGraphics};
-use tokio::io::unix::AsyncFdReadyGuard;
-use tokio::sync::mpsc::UnboundedReceiver;
-use tokio::{
-	io::unix::AsyncFd, net::UnixListener as AsyncUnixListener, sync::mpsc, task::JoinHandle,
+use tokio::{net::UnixStream, sync::mpsc, task::AbortHandle};
+use tokio_stream::StreamExt;
+use tracing::{debug_span, instrument};
+use vulkano_data::setup_vulkano_context;
+use waynest::{
+	server::{
+		self,
+		protocol::{
+			core::wayland::{wl_buffer::WlBuffer, wl_callback::WlCallback, wl_display::WlDisplay},
+			stable::xdg_shell::xdg_toplevel::XdgToplevel,
+		},
+	},
+	wire::{DecodeError, ObjectId},
 };
-use tracing::{debug_span, info, instrument};
+use xdg::toplevel::Toplevel;
 
-pub static WAYLAND_DISPLAY: OnceCell<String> = OnceCell::new();
+pub static WAYLAND_DISPLAY: OnceLock<PathBuf> = OnceLock::new();
 
-struct EGLRawHandles {
-	display: *const c_void,
-	config: *const c_void,
-	context: *const c_void,
-}
-fn get_sk_egl() -> Result<EGLRawHandles> {
-	ensure!(
-		Backend::graphics() == BackendGraphics::OpenGLESEGL,
-		"StereoKit is not running using EGL!"
-	);
-
-	Ok(unsafe {
-		EGLRawHandles {
-			display: stereokit_rust::system::backend_opengl_egl_get_display() as *const c_void,
-			config: stereokit_rust::system::backend_opengl_egl_get_config() as *const c_void,
-			context: stereokit_rust::system::backend_opengl_egl_get_context() as *const c_void,
-		}
-	})
-}
-
-pub struct DisplayWrapper(Mutex<Display<WaylandState>>, DisplayHandle);
-impl DisplayWrapper {
-	pub fn handle(&self) -> DisplayHandle {
-		self.1.clone()
-	}
-	pub fn dispatch_clients(&self, state: &mut WaylandState) -> Result<usize, std::io::Error> {
-		self.0.lock().dispatch_clients(state)
-	}
-	pub fn flush_clients(&self, client: Option<ClientId>) {
-		if let Some(mut lock) = self.0.try_lock() {
-			let _ = lock.backend().flush(client);
-		}
-	}
-	pub fn poll_fd(&self) -> Result<OwnedFd, std::io::Error> {
-		self.0.lock().backend().poll_fd().try_clone_to_owned()
+impl From<waynest::server::Error> for ServerError {
+	fn from(err: waynest::server::Error) -> Self {
+		ServerError::WaylandError(err)
 	}
 }
 
-struct UnownedFd(Option<AsyncFd<OwnedFd>>);
-impl UnownedFd {
-	async fn readable(&self) -> std::io::Result<AsyncFdReadyGuard<'_, OwnedFd>> {
-		self.0.as_ref().unwrap().readable().await
+pub fn get_free_wayland_socket_path() -> Option<(PathBuf, FlockLock<File>)> {
+	// Use XDG runtime directory for secure, user-specific sockets
+	let base_dirs = directories::BaseDirs::new()?;
+	let runtime_dir = base_dirs.runtime_dir()?;
+
+	// Iterate through conventional display numbers (matches X11 behavior)
+	for display in 0..=32 {
+		let socket_path = runtime_dir.join(format!("wayland-{display}"));
+		let socket_lock_path = runtime_dir.join(format!("wayland-{display}.lock"));
+
+		// Open lock file without truncation to preserve existing locks
+		let Ok(lock) = OpenOptions::new()
+			.create(true)
+			.truncate(false) // Prevent destroying other processes' locks
+			.read(true)
+			.write(true)
+			.mode(0o660) // Match Wayland-compositor permissions
+			.open(&socket_lock_path)
+		else {
+			continue;
+		};
+
+		// Atomic mutual exclusion: fail if another process holds the lock\
+		let Ok(lock) = lock.try_exclusive_lock() else {
+			continue; // Lock held by active compositor
+		};
+
+		// Check for zombie sockets (file exists but nothing listening)
+		if socket_path.exists() {
+			match std::os::unix::net::UnixStream::connect(&socket_path) {
+				Ok(_) => continue, // Active compositor found - skip
+				Err(e) if e.kind() == ErrorKind::ConnectionRefused => {
+					// Stale socket - safe to remove since we hold the lock
+					let _ = fs::remove_file(&socket_path);
+				}
+				Err(_) => continue, // Transient error - conservative skip
 			}
 		}
-impl Drop for UnownedFd {
+
+		// Found viable candidate: lock held, socket cleared/available
+		return Some((socket_path, lock));
+	}
+
+	None // Exhausted all conventional display numbers
+}
+
+pub enum Message {
+	Disconnect,
+	Frame(Arc<Callback>),
+	ReleaseBuffer(Arc<Buffer>),
+	CloseToplevel(Arc<Toplevel>),
+	ResizeToplevel {
+		toplevel: Arc<Toplevel>,
+		size: Option<Vector2<u32>>,
+	},
+	SetToplevelVisualActive {
+		toplevel: Arc<Toplevel>,
+		active: bool,
+	},
+	Seat(SeatMessage),
+	SendPresentationFeedback {
+		surface: Arc<Surface>,
+		display_timestamp: MonotonicTimestamp,
+		refresh_cycle: u64,
+	},
+}
+
+pub type MessageSink = mpsc::UnboundedSender<Message>;
+
+#[derive(Debug)]
+struct WaylandClient {
+	abort_handle: AbortHandle,
+}
+impl WaylandClient {
+	pub fn from_stream(socket: UnixStream) -> Result<Self> {
+		let pid = socket.peer_cred().ok().and_then(|c| c.pid());
+		let mut client = server::Client::new(socket)?;
+		let (message_sink, message_source) = mpsc::unbounded_channel();
+
+		client.insert(ObjectId::DISPLAY, Display::new(message_sink, pid));
+		let abort_handle = task::new(
+			|| "wayland client",
+			Self::handle_client_messages(client, message_source),
+		)?
+		.abort_handle();
+
+		Ok(WaylandClient { abort_handle })
+	}
+	async fn handle_client_messages(
+		mut client: server::Client,
+		mut render_message_rx: mpsc::UnboundedReceiver<Message>,
+	) -> Result<()> {
+		loop {
+			tokio::select! {
+				// send all queued up messages
+				msg = render_message_rx.recv() => {
+					if let Some(msg) = msg {
+						Self::handle_render_message(&mut client, msg).await?;
+					}
+				}
+				// handle the next message
+				msg = client.next_message() => {
+					match msg {
+						Ok(Some(mut msg)) => {
+							if let Err(e) = client.handle_message(&mut msg).await {
+								tracing::error!("Wayland: Error handling message: {:?}", e);
+								break;
+							}
+						}
+						Err(e) => {
+							// wayland clients really aren't nice when disconnecting properly, are they? :p
+							if let server::Error::Decode(DecodeError::IoError(e)) = &e && e.kind() == io::ErrorKind::ConnectionReset {
+									if let Some(pid) = client.get::<Display>(ObjectId::DISPLAY).and_then(|d| d.pid) {
+										tracing::info!("Wayland: Client with pid: {pid} disconnected from server");
+									} else {
+										tracing::info!("Wayland: Unknown client disconnected from server");
+									}
+									break;
+							}
+							tracing::error!("Wayland: Error reading message: {:?}", e);
+							break;
+						}
+						Ok(None) => {
+							if let Some(pid) = client.get::<Display>(ObjectId::DISPLAY).and_then(|d| d.pid) {
+								tracing::info!("Wayland: Client with pid: {pid} disconnected from server");
+							} else {
+								tracing::info!("Wayland: Unknown client disconnected from server");
+							}
+							// Message stream ended
+							break;
+						}
+					}
+				}
+			}
+		}
+		Ok(())
+	}
+
+	async fn handle_render_message(
+		client: &mut server::Client,
+		message: Message,
+	) -> Result<bool, waynest::server::Error> {
+		match message {
+			Message::Disconnect => return Ok(true),
+			Message::Frame(callback) => {
+				let now = rustix::time::clock_gettime(rustix::time::ClockId::Monotonic);
+				let now = Duration::new(now.tv_sec as u64, now.tv_nsec as u32);
+				let ms = (now.as_millis() % (u32::MAX as u128)) as u32;
+				callback.done(client, callback.0, ms).await?;
+				client
+					.get::<Display>(ObjectId::DISPLAY)
+					.unwrap()
+					.delete_id(client, ObjectId::DISPLAY, callback.0.as_raw())
+					.await?;
+				client.remove(callback.0);
+			}
+			Message::ReleaseBuffer(buffer) => {
+				buffer.release(client, buffer.id).await?;
+			}
+			Message::CloseToplevel(toplevel) => {
+				toplevel.close(client, toplevel.id).await?;
+			}
+			Message::ResizeToplevel { toplevel, size } => {
+				toplevel.set_size(size);
+				toplevel.reconfigure(client).await?;
+			}
+			Message::SetToplevelVisualActive { toplevel, active } => {
+				toplevel.set_activated(active);
+				toplevel.reconfigure(client).await?;
+			}
+			Message::Seat(seat_message) => {
+				if let Some(seat) = client.get::<Display>(ObjectId::DISPLAY).unwrap().seat.get() {
+					seat.handle_message(client, seat_message).await?;
+				}
+			}
+			Message::SendPresentationFeedback {
+				surface,
+				display_timestamp,
+				refresh_cycle,
+			} => {
+				surface
+					.send_presentation_feedback(client, display_timestamp, refresh_cycle)
+					.await?;
+			}
+		}
+		Ok(false)
+	}
+}
+impl Drop for WaylandClient {
 	fn drop(&mut self) {
-		self.0.take().unwrap().into_inner().into_raw_fd();
+		self.abort_handle.abort();
 	}
 }
 
+#[derive(Debug, Resource)]
 pub struct Wayland {
-	display: Arc<DisplayWrapper>,
-	pub socket_name: Option<String>,
-	join_handle: JoinHandle<Result<()>>,
-	renderer: GlesRenderer,
-	output: Output,
-	dmabuf_rx: UnboundedReceiver<(Dmabuf, Option<dmabuf::ImportNotifier>)>,
+	_lockfile: FlockLock<File>,
+	abort_handle: AbortHandle,
 }
 impl Wayland {
 	pub fn new() -> Result<Self> {
-		let egl_raw_handles = get_sk_egl()?;
-		let renderer = unsafe {
-			GlesRenderer::new(EGLContext::from_raw(
-				egl_raw_handles.display,
-				egl_raw_handles.config,
-				egl_raw_handles.context,
-			)?)?
-		};
+		let (socket_path, _lockfile) =
+			get_free_wayland_socket_path().ok_or(ServerError::WaylandError(
+				waynest::server::Error::IoError(std::io::ErrorKind::AddrNotAvailable.into()),
+			))?;
 
-		let display: Display<WaylandState> = Display::new()?;
-		let display_handle = display.handle();
+		let _ = WAYLAND_DISPLAY.set(socket_path.clone());
 
-		let (dmabuf_tx, dmabuf_rx) = mpsc::unbounded_channel();
-		let display = Arc::new(DisplayWrapper(Mutex::new(display), display_handle.clone()));
+		let listener =
+			server::Listener::new_with_path(&socket_path).map_err(ServerError::WaylandError)?;
 
-		let wayland_state = WaylandState::new(display_handle.clone(), &renderer, dmabuf_tx);
-		let output = wayland_state.lock().output.clone();
+		let abort_handle =
+			task::new(|| "wayland loop", Self::handle_wayland_loop(listener))?.abort_handle();
 
-		let socket = ListeningSocket::bind_auto("wayland", 0..33)?;
-		let socket_name = socket
-			.socket_name()
-			.and_then(OsStr::to_str)
-			.map(ToString::to_string);
-		if let Some(socket_name) = &socket_name {
-			let _ = WAYLAND_DISPLAY.set(socket_name.clone());
-		}
-		info!(socket_name, "Wayland active");
-
-		let join_handle = Wayland::start_loop(display.clone(), socket, wayland_state)?;
-
-		Ok(Wayland {
-			display,
-			socket_name,
-			join_handle,
-			renderer,
-			output,
-			dmabuf_rx,
+		Ok(Self {
+			_lockfile,
+			abort_handle,
 		})
 	}
-
-	fn start_loop(
-		display: Arc<DisplayWrapper>,
-		socket: ListeningSocket,
-		state: Arc<Mutex<WaylandState>>,
-	) -> Result<JoinHandle<Result<()>>> {
-		let listen_async =
-			AsyncUnixListener::from_std(unsafe { UnixListener::from_raw_fd(socket.as_raw_fd()) })?;
-
-		let dispatch_poll_fd = display.poll_fd()?;
-		let dispatch_poll_listener = UnownedFd(Some(AsyncFd::new(dispatch_poll_fd)?));
-
-		let dh1 = display.handle();
-		let mut dh2 = dh1.clone();
-
-		task::new(|| "wayland loop", async move {
-			let _socket = socket; // Keep the socket alive
+	async fn handle_wayland_loop(mut listener: server::Listener) -> Result<()> {
+		let mut clients = Vec::new();
 		loop {
-				tokio::select! {
-					acc = listen_async.accept() => { // New client connected
-						let (stream, _) = acc?;
-						let client_state = Arc::new(ClientState {
-							pid: stream.peer_cred().ok().and_then(|c| c.pid()),
-							id: OnceCell::new(),
-							compositor_state: Default::default(),
-							seat: state.lock().seat.clone(),
+			if let Ok(Some(stream)) = listener.try_next().await {
+				debug_span!("Accept wayland client").in_scope(|| {
+					if let Ok(client) = WaylandClient::from_stream(stream) {
+						clients.push(client);
+					}
 				});
-						let _client = dh2.insert_client(stream.into_std()?, client_state.clone())?;
 			}
-					e = dispatch_poll_listener.readable() => { // Dispatch
-						let mut guard = e?;
-						debug_span!("Dispatch wayland event").in_scope(|| -> Result<(), color_eyre::Report> {
-							display.dispatch_clients(&mut state.lock())?;
-							display.flush_clients(None);
+			clients.retain(|client| !client.abort_handle.is_finished());
+		}
+
+		#[allow(unreachable_code)]
 		Ok(())
-						})?;
-						guard.clear_ready();
-					}
-				}
-			}
-		})
-	}
-
-	#[instrument(level = "debug", name = "Wayland frame", skip(self))]
-	pub fn update(&mut self) {
-		while let Ok((dmabuf, notifier)) = self.dmabuf_rx.try_recv() {
-			if self.renderer.import_dmabuf(&dmabuf, None).is_err() {
-				if let Some(notifier) = notifier {
-					notifier.failed();
-				}
-			}
-		}
-		for core_surface in CORE_SURFACES.get_valid_contents() {
-			core_surface.process(&mut self.renderer);
-		}
-		let _ = self.renderer.cleanup_texture_cache();
-
-		self.display.flush_clients(None);
-	}
-
-	pub fn frame_event(&self) {
-		for core_surface in CORE_SURFACES.get_valid_contents() {
-			core_surface.frame(self.output.clone());
-		}
-	}
-
-	pub fn make_context_current(&self) {
-		unsafe {
-			let _ = self.renderer.egl_context().make_current();
-		}
 	}
 }
 impl Drop for Wayland {
 	fn drop(&mut self) {
-		self.join_handle.abort();
+		self.abort_handle.abort();
+	}
+}
+
+static RENDER_DEVICE: OnceLock<RenderDevice> = OnceLock::new();
+
+pub struct WaylandPlugin;
+impl Plugin for WaylandPlugin {
+	fn build(&self, app: &mut App) {
+		app.add_systems(Update, update_graphics.before(ModelNodeSystemSet));
+		app.init_resource::<UsedBuffers>();
+		app.sub_app_mut(RenderApp)
+			.init_resource::<UsedBuffers>()
+			.add_systems(
+				Render,
+				init_render_device.run_if(|| RENDER_DEVICE.get().is_none()),
+			);
+	}
+	fn finish(&self, app: &mut App) {
+		app.sub_app_mut(RenderApp)
+			.add_systems(Render, setup_vulkano_context)
+			.add_systems(Render, before_render.in_set(XrRenderSet::PreRender))
+			.add_systems(Render, after_render.in_set(XrRenderSet::PostRender))
+			.add_systems(
+				Render,
+				submit_frame_timings
+					.in_set(XrRenderSet::PostRender)
+					.after(end_frame),
+			);
+	}
+}
+
+fn init_render_device(dev: Res<RenderDevice>) {
+	_ = RENDER_DEVICE.set(dev.clone());
+}
+
+#[derive(Resource, Deref, DerefMut)]
+struct UsedBuffers(OwnedRegistry<BufferUsage>);
+impl Default for UsedBuffers {
+	fn default() -> Self {
+		Self(OwnedRegistry::new())
+	}
+}
+
+fn before_render(buffers: Res<UsedBuffers>) {
+	for buf in WL_SURFACE_REGISTRY
+		.get_valid_contents()
+		.into_iter()
+		.filter_map(|surface| surface.current_state().buffer)
+		.filter_map(|buffer| buffer.usage)
+	{
+		buffers.add_raw(buf);
+	}
+	for surface in WL_SURFACE_REGISTRY.get_valid_contents() {
+		surface.frame_event();
+	}
+}
+
+fn after_render(buffers: Res<UsedBuffers>) {
+	buffers.clear();
+}
+
+#[instrument(level = "debug", name = "Wayland frame", skip_all)]
+fn update_graphics(
+	dmatexes: Res<ImportedDmatexs>,
+	mut materials: ResMut<Assets<BevyMaterial>>,
+	mut images: ResMut<Assets<Image>>,
+) {
+	for surface in WL_SURFACE_REGISTRY.get_valid_contents() {
+		surface.update_graphics(&dmatexes, &mut materials, &mut images);
+	}
+}
+
+#[instrument(level = "debug", name = "Wayland frame", skip_all)]
+fn submit_frame_timings(mut frame_count: Local<u64>) {
+	*frame_count += 1;
+	for surface in WL_SURFACE_REGISTRY.get_valid_contents() {
+		let display_timestamp =
+			rustix::time::clock_gettime(rustix::time::ClockId::Monotonic).into();
+		surface.submit_presentation_feedback(display_timestamp, *frame_count);
 	}
 }

src/wayland/presentation.rs

@@ -0,0 +1,66 @@
+use rustix::fs::Timespec;
+use waynest::{
+	server::{
+		Client, Dispatcher, Result,
+		protocol::stable::presentation_time::{
+			wp_presentation::WpPresentation, wp_presentation_feedback::WpPresentationFeedback,
+		},
+	},
+	wire::ObjectId,
+};
+
+use crate::wayland::core::surface::Surface;
+
+pub struct MonotonicTimestamp {
+	secs: u64,
+	subsec_nanos: u32,
+}
+
+impl MonotonicTimestamp {
+	pub fn secs_lo(&self) -> u32 {
+		self.secs as u32
+	}
+	pub fn secs_hi(&self) -> u32 {
+		(self.secs >> 16) as u32
+	}
+	pub fn subsec_nanos(&self) -> u32 {
+		self.subsec_nanos
+	}
+}
+impl From<Timespec> for MonotonicTimestamp {
+	fn from(value: Timespec) -> Self {
+		Self {
+			secs: value.tv_sec as u64,
+			subsec_nanos: value.tv_nsec as u32,
+		}
+	}
+}
+
+#[derive(Debug, Dispatcher)]
+pub struct Presentation;
+impl WpPresentation for Presentation {
+	async fn destroy(&self, _client: &mut Client, _sender_id: ObjectId) -> Result<()> {
+		Ok(())
+	}
+
+	async fn feedback(
+		&self,
+		client: &mut Client,
+		_sender_id: ObjectId,
+		surface: ObjectId,
+		id: ObjectId,
+	) -> Result<()> {
+		let Some(surface) = client.get::<Surface>(surface) else {
+			tracing::error!("unable to get surface#{surface}");
+			return Ok(());
+		};
+		let feedback = client.insert(id, PresentationFeedback(id));
+		surface.add_presentation_feedback(feedback);
+
+		Ok(())
+	}
+}
+
+#[derive(Debug, Dispatcher)]
+pub struct PresentationFeedback(pub ObjectId);
+impl WpPresentationFeedback for PresentationFeedback {}

src/wayland/registry.rs

@@ -0,0 +1,209 @@
+use crate::wayland::{
+	core::{
+		compositor::{Compositor, WlCompositor},
+		data_device::DataDeviceManager,
+		output::{Output, WlOutput},
+		seat::{Seat, WlSeat},
+		shm::{Shm, WlShm},
+	},
+	dmabuf::Dmabuf,
+	mesa_drm::MesaDrm,
+	presentation::Presentation,
+	util::ClientExt,
+	xdg::wm_base::{WmBase, XdgWmBase},
+};
+use waynest::{
+	server::{
+		Client, Dispatcher, Error, Result,
+		protocol::{
+			core::wayland::{wl_data_device_manager::WlDataDeviceManager, wl_registry::*},
+			external::drm::wl_drm::WlDrm,
+			stable::{
+				linux_dmabuf_v1::zwp_linux_dmabuf_v1::ZwpLinuxDmabufV1,
+				presentation_time::wp_presentation::WpPresentation,
+			},
+		},
+	},
+	wire::{NewId, ObjectId},
+};
+
+struct RegistryGlobals;
+impl RegistryGlobals {
+	pub const COMPOSITOR: u32 = 0;
+	pub const SHM: u32 = 1;
+	pub const WM_BASE: u32 = 2;
+	pub const SEAT: u32 = 3;
+	pub const DATA_DEVICE_MANAGER: u32 = 4;
+	pub const OUTPUT: u32 = 5;
+	pub const DMABUF: u32 = 6;
+	pub const WL_DRM: u32 = 7;
+	pub const PRESENTATION: u32 = 8;
+}
+
+#[derive(Debug, Dispatcher, Default)]
+pub struct Registry;
+
+impl Registry {
+	pub async fn advertise_globals(&self, client: &mut Client, sender_id: ObjectId) -> Result<()> {
+		self.global(
+			client,
+			sender_id,
+			RegistryGlobals::COMPOSITOR,
+			Compositor::INTERFACE.to_string(),
+			Compositor::VERSION,
+		)
+		.await?;
+
+		self.global(
+			client,
+			sender_id,
+			RegistryGlobals::SHM,
+			Shm::INTERFACE.to_string(),
+			Shm::VERSION,
+		)
+		.await?;
+
+		self.global(
+			client,
+			sender_id,
+			RegistryGlobals::WM_BASE,
+			WmBase::INTERFACE.to_string(),
+			WmBase::VERSION,
+		)
+		.await?;
+
+		self.global(
+			client,
+			sender_id,
+			RegistryGlobals::SEAT,
+			Seat::INTERFACE.to_string(),
+			Seat::VERSION,
+		)
+		.await?;
+
+		self.global(
+			client,
+			sender_id,
+			RegistryGlobals::DATA_DEVICE_MANAGER,
+			DataDeviceManager::INTERFACE.to_string(),
+			DataDeviceManager::VERSION,
+		)
+		.await?;
+
+		self.global(
+			client,
+			sender_id,
+			RegistryGlobals::OUTPUT,
+			Output::INTERFACE.to_string(),
+			Output::VERSION,
+		)
+		.await?;
+
+		self.global(
+			client,
+			sender_id,
+			RegistryGlobals::DMABUF,
+			crate::wayland::dmabuf::Dmabuf::INTERFACE.to_string(),
+			Dmabuf::VERSION,
+		)
+		.await?;
+
+		self.global(
+			client,
+			sender_id,
+			RegistryGlobals::WL_DRM,
+			crate::wayland::mesa_drm::MesaDrm::INTERFACE.to_string(),
+			MesaDrm::VERSION,
+		)
+		.await?;
+
+		self.global(
+			client,
+			sender_id,
+			RegistryGlobals::PRESENTATION,
+			Presentation::INTERFACE.to_string(),
+			Presentation::VERSION,
+		)
+		.await?;
+
+		Ok(())
+	}
+}
+
+impl WlRegistry for Registry {
+	async fn bind(
+		&self,
+		client: &mut Client,
+		_sender_id: ObjectId,
+		name: u32,
+		new_id: NewId,
+	) -> Result<()> {
+		match name {
+			RegistryGlobals::COMPOSITOR => {
+				tracing::info!("Binding compositor");
+				client.insert(new_id.object_id, Compositor);
+			}
+			RegistryGlobals::SHM => {
+				tracing::info!("Binding SHM");
+				let shm = client.insert(new_id.object_id, Shm);
+				shm.advertise_formats(client, new_id.object_id).await?;
+			}
+			RegistryGlobals::WM_BASE => {
+				tracing::info!("Binding WM_BASE");
+				client.insert(
+					new_id.object_id,
+					WmBase {
+						version: new_id.version,
+					},
+				);
+			}
+			RegistryGlobals::SEAT => {
+				tracing::info!("Binding seat with id {}", new_id.object_id);
+				let seat = Seat::new(client, new_id.object_id, new_id.version).await?;
+				let seat = client.insert(new_id.object_id, seat);
+				let _ = client.display().seat.set(seat.clone());
+
+				tracing::info!("Seat capabilities advertised");
+			}
+			RegistryGlobals::DATA_DEVICE_MANAGER => {
+				tracing::info!("Binding data device manager");
+				client.insert(new_id.object_id, DataDeviceManager);
+			}
+			RegistryGlobals::OUTPUT => {
+				tracing::info!("Binding output");
+				let output = client.insert(
+					new_id.object_id,
+					Output {
+						id: new_id.object_id,
+						version: new_id.version,
+					},
+				);
+				let _ = client.display().output.set(output.clone());
+				output.advertise_outputs(client).await?;
+			}
+			RegistryGlobals::DMABUF => {
+				tracing::info!("Binding dmabuf");
+
+				let dmabuf = Dmabuf::new(client, new_id.object_id, new_id.version).await?;
+				client.insert(new_id.object_id, dmabuf);
+			}
+			RegistryGlobals::WL_DRM => {
+				tracing::info!("Binding wl_drm");
+
+				let drm = MesaDrm::new(client, new_id.object_id, new_id.version).await?;
+				client.insert(new_id.object_id, drm);
+			}
+			RegistryGlobals::PRESENTATION => {
+				tracing::info!("Binding wp_presentation");
+
+				client.insert(new_id.object_id, Presentation);
+			}
+			id => {
+				tracing::error!(id, "Wayland: failed to bind to registry global");
+				return Err(Error::MissingObject(unsafe { ObjectId::from_raw(name) }));
+			}
+		}
+
+		Ok(())
+	}
+}

src/wayland/seat.rs

@@ -1,307 +0,0 @@
-use super::{state::WaylandState, surface::CoreSurface, utils::WlSurfaceExt};
-use crate::{
-	core::task,
-	nodes::{
-		data::KEYMAPS,
-		items::panel::{Backend, Geometry, PanelItem},
-	},
-};
-use mint::Vector2;
-use parking_lot::Mutex;
-use rustc_hash::FxHashMap;
-use slotmap::KeyData;
-use smithay::{
-	backend::input::{AxisRelativeDirection, ButtonState, KeyState},
-	delegate_seat,
-	input::{
-		keyboard::{FilterResult, LedState},
-		pointer::{AxisFrame, ButtonEvent, CursorImageStatus, MotionEvent},
-		touch::{self, DownEvent, UpEvent},
-		Seat, SeatHandler,
-	},
-	reexports::wayland_server::{protocol::wl_surface::WlSurface, Resource, Weak as WlWeak},
-	utils::SERIAL_COUNTER,
-	wayland::compositor,
-};
-use std::sync::{Arc, Weak};
-use tokio::sync::watch;
-
-impl SeatHandler for WaylandState {
-	type PointerFocus = WlSurface;
-	type KeyboardFocus = WlSurface;
-	type TouchFocus = WlSurface;
-
-	fn seat_state(&mut self) -> &mut smithay::input::SeatState<Self> {
-		&mut self.seat_state
-	}
-	fn focus_changed(&mut self, _seat: &Seat<Self>, _focused: Option<&Self::KeyboardFocus>) {}
-	fn cursor_image(&mut self, _seat: &Seat<Self>, image: CursorImageStatus) {
-		self.seat.cursor_info_tx.send_modify(|c| match image {
-			CursorImageStatus::Hidden => c.surface = None,
-			CursorImageStatus::Surface(surface) => {
-				CoreSurface::add_to(&surface);
-				compositor::with_states(&surface, |data| {
-					if let Some(core_surface) = data.data_map.get::<Arc<CoreSurface>>() {
-						core_surface.set_material_offset(1);
-					}
-				});
-				c.surface = Some(surface.downgrade())
-			}
-			_ => (),
-		});
-	}
-	fn led_state_changed(&mut self, _seat: &Seat<Self>, _led_state: LedState) {}
-}
-delegate_seat!(WaylandState);
-
-pub fn handle_cursor<B: Backend>(
-	panel_item: &Arc<PanelItem<B>>,
-	mut cursor: watch::Receiver<CursorInfo>,
-) {
-	let panel_item_weak = Arc::downgrade(panel_item);
-	let _ = task::new(|| "cursor handler", async move {
-		while cursor.changed().await.is_ok() {
-			let Some(panel_item) = panel_item_weak.upgrade() else {
-				continue;
-			};
-			let cursor_info = cursor.borrow();
-			panel_item.set_cursor(cursor_info.cursor_data());
-		}
-	});
-}
-pub struct CursorInfo {
-	pub surface: Option<WlWeak<WlSurface>>,
-	pub hotspot_x: i32,
-	pub hotspot_y: i32,
-}
-impl CursorInfo {
-	pub fn cursor_data(&self) -> Option<Geometry> {
-		let cursor_size = self.surface.as_ref()?.upgrade().ok()?.get_size()?;
-		Some(Geometry {
-			origin: [self.hotspot_x, self.hotspot_y].into(),
-			size: cursor_size,
-		})
-	}
-}
-
-pub struct SeatWrapper {
-	wayland_state: Weak<Mutex<WaylandState>>,
-	cursor_info_tx: watch::Sender<CursorInfo>,
-	pub cursor_info_rx: watch::Receiver<CursorInfo>,
-	seat: Seat<WaylandState>,
-	touches: Mutex<FxHashMap<u32, WlWeak<WlSurface>>>,
-}
-impl SeatWrapper {
-	pub fn new(wayland_state: Weak<Mutex<WaylandState>>, seat: Seat<WaylandState>) -> Self {
-		let (cursor_info_tx, cursor_info_rx) = watch::channel(CursorInfo {
-			surface: None,
-			hotspot_x: 0,
-			hotspot_y: 0,
-		});
-		SeatWrapper {
-			wayland_state,
-			cursor_info_tx,
-			cursor_info_rx,
-			seat,
-			touches: Mutex::new(FxHashMap::default()),
-		}
-	}
-	pub fn unfocus(&self, surface: &WlSurface, state: &mut WaylandState) {
-		let pointer = self.seat.get_pointer().unwrap();
-		if pointer.current_focus() == Some(surface.clone()) {
-			pointer.motion(
-				state,
-				None,
-				&MotionEvent {
-					location: (0.0, 0.0).into(),
-					serial: SERIAL_COUNTER.next_serial(),
-					time: 0,
-				},
-			)
-		}
-		let keyboard = self.seat.get_keyboard().unwrap();
-		if keyboard.current_focus() == Some(surface.clone()) {
-			keyboard.set_focus(state, None, SERIAL_COUNTER.next_serial());
-		}
-		for (id, touch_surface) in self.touches.lock().iter() {
-			if touch_surface.id() == surface.id() {
-				self.touch_up(*id);
-			}
-		}
-	}
-
-	pub fn pointer_motion(&self, surface: WlSurface, position: Vector2<f32>) {
-		let Some(state) = self.wayland_state.upgrade() else {
-			return;
-		};
-		let mut state = state.lock();
-		let Some(pointer) = self.seat.get_pointer() else {
-			return;
-		};
-		pointer.motion(
-			&mut state,
-			Some((surface, (0.0, 0.0).into())),
-			&MotionEvent {
-				location: (position.x as f64, position.y as f64).into(),
-				serial: SERIAL_COUNTER.next_serial(),
-				time: 0,
-			},
-		);
-		pointer.frame(&mut state);
-	}
-	pub fn pointer_button(&self, button: u32, pressed: bool) {
-		let Some(state) = self.wayland_state.upgrade() else {
-			return;
-		};
-		let mut state = state.lock();
-		let Some(pointer) = self.seat.get_pointer() else {
-			return;
-		};
-		pointer.button(
-			&mut state,
-			&ButtonEvent {
-				button,
-				state: if pressed {
-					ButtonState::Pressed
-				} else {
-					ButtonState::Released
-				},
-				serial: SERIAL_COUNTER.next_serial(),
-				time: 0,
-			},
-		);
-		pointer.frame(&mut state);
-	}
-	pub fn pointer_scroll(
-		&self,
-		scroll_distance: Option<Vector2<f32>>,
-		scroll_steps: Option<Vector2<f32>>,
-	) {
-		let Some(state) = self.wayland_state.upgrade() else {
-			return;
-		};
-		let mut state = state.lock();
-		let Some(pointer) = self.seat.get_pointer() else {
-			return;
-		};
-		pointer.axis(
-			&mut state,
-			AxisFrame {
-				source: None,
-				relative_direction: (
-					AxisRelativeDirection::Identical,
-					AxisRelativeDirection::Identical,
-				),
-				time: 0,
-				axis: scroll_distance
-					.map(|d| (d.x as f64, d.y as f64))
-					.unwrap_or((0.0, 0.0)),
-				v120: scroll_steps.map(|d| ((d.x * 120.0) as i32, (d.y * 120.0) as i32)),
-				stop: (false, false),
-			},
-		);
-		pointer.frame(&mut state);
-	}
-
-	pub fn keyboard_keys(&self, surface: WlSurface, keymap_id: u64, keys: Vec<i32>) {
-		let Some(state) = self.wayland_state.upgrade() else {
-			return;
-		};
-		let Some(keyboard) = self.seat.get_keyboard() else {
-			return;
-		};
-		let keymaps = KEYMAPS.lock();
-		let Some(keymap) = keymaps.get(KeyData::from_ffi(keymap_id).into()).cloned() else {
-			return;
-		};
-
-		keyboard.set_focus(
-			&mut state.lock(),
-			Some(surface),
-			SERIAL_COUNTER.next_serial(),
-		);
-		if keyboard
-			.set_keymap_from_string(&mut state.lock(), keymap)
-			.is_err()
-		{
-			return;
-		}
-		for key in keys {
-			keyboard.input(
-				&mut state.lock(),
-				key.unsigned_abs(),
-				if key > 0 {
-					KeyState::Pressed
-				} else {
-					KeyState::Released
-				},
-				SERIAL_COUNTER.next_serial(),
-				0,
-				|_, _, _| FilterResult::Forward::<()>,
-			);
-		}
-	}
-
-	pub fn touch_down(&self, surface: WlSurface, id: u32, position: Vector2<f32>) {
-		let Some(state) = self.wayland_state.upgrade() else {
-			return;
-		};
-		let Some(touch) = self.seat.get_touch() else {
-			return;
-		};
-		touch.down(
-			&mut state.lock(),
-			Some((surface, (0.0, 0.0).into())),
-			&DownEvent {
-				slot: Some(id).into(),
-				location: (position.x as f64, position.y as f64).into(),
-				serial: SERIAL_COUNTER.next_serial(),
-				time: 0,
-			},
-		);
-		touch.frame(&mut state.lock());
-	}
-	pub fn touch_move(&self, id: u32, position: Vector2<f32>) {
-		let Some(state) = self.wayland_state.upgrade() else {
-			return;
-		};
-		let Some(surface) = self.touches.lock().get(&id).and_then(|c| c.upgrade().ok()) else {
-			return;
-		};
-		let Some(touch) = self.seat.get_touch() else {
-			return;
-		};
-		touch.motion(
-			&mut state.lock(),
-			Some((surface, (0.0, 0.0).into())),
-			&touch::MotionEvent {
-				slot: Some(id).into(),
-				location: (position.x as f64, position.y as f64).into(),
-				time: 0,
-			},
-		);
-		touch.frame(&mut state.lock());
-	}
-	pub fn touch_up(&self, id: u32) {
-		let Some(state) = self.wayland_state.upgrade() else {
-			return;
-		};
-		let Some(touch) = self.seat.get_touch() else {
-			return;
-		};
-		touch.up(
-			&mut state.lock(),
-			&UpEvent {
-				slot: Some(id).into(),
-				serial: SERIAL_COUNTER.next_serial(),
-				time: 0,
-			},
-		);
-		touch.frame(&mut state.lock());
-	}
-	pub fn reset_input(&self) {
-		for id in self.touches.lock().keys() {
-			self.touch_up(*id)
-		}
-	}
-}

src/wayland/state.rs

@@ -1,222 +0,0 @@
-use super::seat::SeatWrapper;
-use crate::wayland::drm::wl_drm::WlDrm;
-use once_cell::sync::OnceCell;
-use parking_lot::Mutex;
-use smithay::{
-	backend::{
-		allocator::{dmabuf::Dmabuf, Fourcc},
-		egl::EGLDevice,
-		renderer::gles::GlesRenderer,
-	},
-	delegate_dmabuf, delegate_output, delegate_shm,
-	input::{keyboard::XkbConfig, SeatState},
-	output::{Mode, Output, Scale, Subpixel},
-	reexports::{
-		wayland_protocols::xdg::{
-			decoration::zv1::server::zxdg_decoration_manager_v1::ZxdgDecorationManagerV1,
-			shell::server::xdg_toplevel::WmCapabilities,
-		},
-		wayland_protocols_misc::server_decoration::server::org_kde_kwin_server_decoration_manager::Mode as DecorationMode,
-		wayland_server::{
-			backend::{ClientData, ClientId, DisconnectReason},
-			protocol::{
-				wl_buffer::WlBuffer, wl_data_device_manager::WlDataDeviceManager,
-				wl_output::WlOutput,
-			},
-			DisplayHandle,
-		},
-	},
-	utils::{Size, Transform},
-	wayland::{
-		buffer::BufferHandler,
-		compositor::{CompositorClientState, CompositorState},
-		dmabuf::{
-			self, DmabufFeedback, DmabufFeedbackBuilder, DmabufGlobal, DmabufHandler, DmabufState,
-		},
-		output::OutputHandler,
-		shell::{
-			kde::decoration::KdeDecorationState,
-			xdg::{WmCapabilitySet, XdgShellState},
-		},
-		shm::{ShmHandler, ShmState},
-	},
-};
-use std::sync::Arc;
-use tokio::sync::mpsc::UnboundedSender;
-use tracing::{info, warn};
-
-pub struct ClientState {
-	pub pid: Option<i32>,
-	pub id: OnceCell<ClientId>,
-	pub compositor_state: CompositorClientState,
-	pub seat: Arc<SeatWrapper>,
-}
-impl ClientData for ClientState {
-	fn initialized(&self, client_id: ClientId) {
-		info!("Wayland client {:?} connected", client_id);
-		let _ = self.id.set(client_id);
-	}
-
-	fn disconnected(&self, client_id: ClientId, reason: DisconnectReason) {
-		info!(
-			"Wayland client {:?} disconnected because {:#?}",
-			client_id, reason
-		);
-	}
-}
-
-pub struct WaylandState {
-	pub compositor_state: CompositorState,
-	// pub xdg_activation_state: XdgActivationState,
-	pub kde_decoration_state: KdeDecorationState,
-	pub shm_state: ShmState,
-	dmabuf_state: (DmabufState, DmabufGlobal, Option<DmabufFeedback>),
-	pub drm_formats: Vec<Fourcc>,
-	pub dmabuf_tx: UnboundedSender<(Dmabuf, Option<dmabuf::ImportNotifier>)>,
-	pub seat_state: SeatState<Self>,
-	pub seat: Arc<SeatWrapper>,
-	pub xdg_shell: XdgShellState,
-	pub output: Output,
-}
-
-impl WaylandState {
-	pub fn new(
-		display_handle: DisplayHandle,
-		renderer: &GlesRenderer,
-		dmabuf_tx: UnboundedSender<(Dmabuf, Option<dmabuf::ImportNotifier>)>,
-	) -> Arc<Mutex<Self>> {
-		let compositor_state = CompositorState::new::<Self>(&display_handle);
-		// let xdg_activation_state = XdgActivationState::new::<Self, _>(&display_handle);
-		let kde_decoration_state =
-			KdeDecorationState::new::<Self>(&display_handle, DecorationMode::Server);
-		let shm_state = ShmState::new::<Self>(&display_handle, vec![]);
-		let render_node = EGLDevice::device_for_display(renderer.egl_context().display())
-			.and_then(|device| device.try_get_render_node());
-		let dmabuf_formats = renderer
-			.egl_context()
-			.dmabuf_render_formats()
-			.iter()
-			.cloned()
-			.collect::<Vec<_>>();
-		let drm_formats = dmabuf_formats.iter().map(|f| f.code).collect();
-
-		let dmabuf_default_feedback = match render_node {
-			Ok(Some(node)) => DmabufFeedbackBuilder::new(node.dev_id(), dmabuf_formats.clone())
-				.build()
-				.ok(),
-			Ok(None) => {
-				warn!("failed to query render node, dmabuf will use v3");
-				None
-			}
-			Err(err) => {
-				warn!(?err, "failed to egl device for display, dmabuf will use v3");
-				None
-			}
-		};
-		// if we failed to build dmabuf feedback we fall back to dmabuf v3
-		// Note: egl on Mesa requires either v4 or wl_drm (initialized with bind_wl_display)
-		let dmabuf_state = if let Some(default_feedback) = dmabuf_default_feedback {
-			let mut dmabuf_state = DmabufState::new();
-			let dmabuf_global = dmabuf_state.create_global_with_default_feedback::<WaylandState>(
-				&display_handle,
-				&default_feedback,
-			);
-			(dmabuf_state, dmabuf_global, Some(default_feedback))
-		} else {
-			let mut dmabuf_state = DmabufState::new();
-			let dmabuf_global =
-				dmabuf_state.create_global::<WaylandState>(&display_handle, dmabuf_formats.clone());
-			(dmabuf_state, dmabuf_global, None)
-		};
-
-		let mut seat_state = SeatState::new();
-		let mut seat = seat_state.new_wl_seat(&display_handle, "seat0");
-		seat.add_pointer();
-		seat.add_keyboard(XkbConfig::default(), 200, 25).unwrap();
-		seat.add_touch();
-
-		let output = Output::new(
-			"1x".to_owned(),
-			smithay::output::PhysicalProperties {
-				size: Size::default(),
-				subpixel: Subpixel::None,
-				make: "Virtual XR Display".to_owned(),
-				model: "Your Headset Name Here".to_owned(),
-			},
-		);
-		let _output_global = output.create_global::<Self>(&display_handle);
-		let mode = Mode {
-			size: (1024, 1024).into(),
-			refresh: 60000,
-		};
-		output.change_current_state(
-			Some(mode),
-			Some(Transform::Normal),
-			Some(Scale::Integer(2)),
-			None,
-		);
-		output.set_preferred(mode);
-
-		let mut xdg_shell = XdgShellState::new::<Self>(&display_handle);
-		let mut capabilities = WmCapabilitySet::default();
-		capabilities.set(WmCapabilities::Maximize);
-		capabilities.set(WmCapabilities::Fullscreen);
-		capabilities.unset(WmCapabilities::Minimize);
-		capabilities.unset(WmCapabilities::WindowMenu);
-		xdg_shell.replace_capabilities(capabilities);
-		display_handle.create_global::<Self, WlDataDeviceManager, _>(3, ());
-		display_handle.create_global::<Self, ZxdgDecorationManagerV1, _>(1, ());
-		display_handle.create_global::<Self, WlDrm, _>(2, ());
-
-		info!("Init Wayland compositor");
-
-		Arc::new_cyclic(|weak| {
-			Mutex::new(WaylandState {
-				compositor_state,
-				// xdg_activation_state,
-				kde_decoration_state,
-				shm_state,
-				drm_formats,
-				dmabuf_state,
-				dmabuf_tx,
-				seat_state,
-				seat: Arc::new(SeatWrapper::new(weak.clone(), seat)),
-				xdg_shell,
-				output,
-			})
-		})
-	}
-}
-impl Drop for WaylandState {
-	fn drop(&mut self) {
-		info!("Cleanly shut down the Wayland compositor");
-	}
-}
-impl BufferHandler for WaylandState {
-	fn buffer_destroyed(&mut self, _buffer: &WlBuffer) {}
-}
-impl ShmHandler for WaylandState {
-	fn shm_state(&self) -> &ShmState {
-		&self.shm_state
-	}
-}
-impl DmabufHandler for WaylandState {
-	fn dmabuf_state(&mut self) -> &mut DmabufState {
-		&mut self.dmabuf_state.0
-	}
-
-	fn dmabuf_imported(
-		&mut self,
-		_global: &DmabufGlobal,
-		dmabuf: Dmabuf,
-		notifier: dmabuf::ImportNotifier,
-	) {
-		self.dmabuf_tx.send((dmabuf, Some(notifier))).unwrap();
-	}
-}
-impl OutputHandler for WaylandState {
-	fn output_bound(&mut self, _output: Output, _wl_output: WlOutput) {}
-}
-delegate_dmabuf!(WaylandState);
-delegate_shm!(WaylandState);
-delegate_output!(WaylandState);

src/wayland/surface.rs

@@ -1,206 +0,0 @@
-use super::utils::WlSurfaceExt;
-use crate::{
-	core::{delta::Delta, destroy_queue, registry::Registry},
-	nodes::{
-		drawable::{
-			model::{MaterialWrapper, ModelPart},
-			shaders::PANEL_SHADER_BYTES,
-		},
-		items::camera::TexWrapper,
-	},
-};
-use once_cell::sync::OnceCell;
-use parking_lot::Mutex;
-use send_wrapper::SendWrapper;
-use smithay::{
-	backend::renderer::{
-		gles::{GlesRenderer, GlesTexture},
-		utils::{import_surface_tree, RendererSurfaceStateUserData},
-		Renderer, Texture,
-	},
-	desktop::utils::send_frames_surface_tree,
-	output::Output,
-	reexports::wayland_server::{self, protocol::wl_surface::WlSurface, Resource},
-};
-use std::{ffi::c_void, sync::Arc, time::Duration};
-use stereokit_rust::{
-	material::{Material, Transparency},
-	shader::Shader,
-	tex::{Tex, TexAddress, TexFormat, TexSample, TexType},
-	util::Time,
-};
-
-pub static CORE_SURFACES: Registry<CoreSurface> = Registry::new();
-
-pub struct CoreSurfaceData {
-	wl_tex: Option<SendWrapper<GlesTexture>>,
-}
-impl Drop for CoreSurfaceData {
-	fn drop(&mut self) {
-		destroy_queue::add(self.wl_tex.take());
-	}
-}
-
-pub struct CoreSurface {
-	pub weak_surface: wayland_server::Weak<WlSurface>,
-	mapped_data: Mutex<Option<CoreSurfaceData>>,
-	sk_tex: OnceCell<Mutex<TexWrapper>>,
-	sk_mat: OnceCell<Mutex<MaterialWrapper>>,
-	material_offset: Mutex<Delta<u32>>,
-	pub pending_material_applications: Registry<ModelPart>,
-}
-
-impl CoreSurface {
-	pub fn add_to(surface: &WlSurface) {
-		let core_surface = CORE_SURFACES.add(CoreSurface {
-			weak_surface: surface.downgrade(),
-			mapped_data: Mutex::new(None),
-			sk_tex: OnceCell::new(),
-			sk_mat: OnceCell::new(),
-			material_offset: Mutex::new(Delta::new(0)),
-			pending_material_applications: Registry::new(),
-		});
-		surface.insert_data(core_surface);
-	}
-
-	pub fn from_wl_surface(surf: &WlSurface) -> Option<Arc<CoreSurface>> {
-		surf.get_data()
-	}
-
-	pub fn process(&self, renderer: &mut GlesRenderer) {
-		let Some(wl_surface) = self.wl_surface() else {
-			return;
-		};
-
-		let sk_tex = self.sk_tex.get_or_init(|| {
-			Mutex::new(TexWrapper(Tex::new(
-				TexType::ImageNomips,
-				TexFormat::RGBA32Linear,
-				nanoid::nanoid!(),
-			)))
-		});
-		self.sk_mat.get_or_init(|| {
-			let shader = Shader::from_memory(PANEL_SHADER_BYTES).unwrap();
-			// let _ = renderer.with_context(|c| unsafe {
-			// 	shader_inject(c, &mut shader, SIMULA_VERT_STR, SIMULA_FRAG_STR)
-			// });
-
-			let mut mat = Material::new(shader, None);
-			mat.diffuse_tex(&sk_tex.lock().0);
-			mat.transparency(Transparency::Blend);
-			Mutex::new(MaterialWrapper(mat))
-		});
-
-		// Import all surface buffers into textures
-		if import_surface_tree(renderer, &wl_surface).is_err() {
-			return;
-		}
-
-		self.update_textures(renderer);
-		self.apply_surface_materials();
-	}
-
-	pub fn update_textures(&self, renderer: &mut GlesRenderer) {
-		let Some(wl_surface) = self.wl_surface() else {
-			return;
-		};
-		let mapped = wl_surface
-			.get_data_raw::<RendererSurfaceStateUserData, _, _>(|surface_states| {
-				surface_states.lock().unwrap().buffer().is_some()
-			})
-			.unwrap_or(false);
-
-		if !mapped {
-			return;
-		}
-
-		let mut mapped_data = self.mapped_data.lock();
-
-		let Some(smithay_tex) = wl_surface
-			.get_data_raw::<RendererSurfaceStateUserData, _, _>(|surface_states| {
-				surface_states
-					.lock()
-					.unwrap()
-					.texture::<GlesRenderer>(renderer.id())
-					.cloned()
-			})
-			.flatten()
-		else {
-			return;
-		};
-
-		let Some(sk_tex) = self.sk_tex.get() else {
-			return;
-		};
-		let Some(sk_mat) = self.sk_mat.get() else {
-			return;
-		};
-		sk_tex
-			.lock()
-			.0
-			.set_native_surface(
-				smithay_tex.tex_id() as usize as *mut c_void,
-				TexType::ImageNomips,
-				smithay::backend::renderer::gles::ffi::RGBA8.into(),
-				smithay_tex.width() as i32,
-				smithay_tex.height() as i32,
-				1,
-				false,
-			)
-			.sample_mode(TexSample::Point)
-			.address_mode(TexAddress::Clamp);
-
-		if let Some(material_offset) = self.material_offset.lock().delta() {
-			sk_mat.lock().0.queue_offset(*material_offset as i32);
-		}
-
-		let new_mapped_data = CoreSurfaceData {
-			wl_tex: Some(SendWrapper::new(smithay_tex)),
-		};
-		*mapped_data = Some(new_mapped_data);
-	}
-
-	pub fn frame(&self, output: Output) {
-		let Some(wl_surface) = self.wl_surface() else {
-			return;
-		};
-
-		send_frames_surface_tree(
-			&wl_surface,
-			&output,
-			Duration::from_secs_f64(Time::get_total_unscaled()),
-			None,
-			|_, _| Some(output.clone()),
-		);
-	}
-
-	pub fn set_material_offset(&self, material_offset: u32) {
-		*self.material_offset.lock().value_mut() = material_offset;
-	}
-
-	pub fn apply_material(&self, model_part: &Arc<ModelPart>) {
-		self.pending_material_applications.add_raw(model_part)
-	}
-
-	fn apply_surface_materials(&self) {
-		if let Some(sk_mat) = self.sk_mat.get() {
-			let sk_mat = sk_mat.lock();
-			for model_node in self.pending_material_applications.get_valid_contents() {
-				model_node.replace_material_now(&sk_mat.0);
-			}
-			self.pending_material_applications.clear();
-		}
-	}
-
-	pub fn wl_surface(&self) -> Option<WlSurface> {
-		self.weak_surface.upgrade().ok()
-	}
-}
-impl Drop for CoreSurface {
-	fn drop(&mut self) {
-		CORE_SURFACES.remove(self);
-
-		destroy_queue::add(self.sk_tex.take());
-		destroy_queue::add(self.sk_mat.take());
-	}
-}

src/wayland/util.rs

@@ -0,0 +1,67 @@
+#![allow(unused)]
+
+use super::{Message, MessageSink, display::Display};
+use std::{fmt::Debug, sync::Arc};
+use waynest::{
+	server::{Client, Result, protocol::core::wayland::wl_display::WlDisplay},
+	wire::ObjectId,
+};
+
+pub trait ClientExt {
+	fn message_sink(&self) -> MessageSink;
+	fn display(&self) -> Arc<Display>;
+	async fn protocol_error(
+		&mut self,
+		sender_id: ObjectId,
+		object_id: ObjectId,
+		code: u32,
+		message: String,
+	) -> Result<()>;
+}
+impl ClientExt for Client {
+	fn message_sink(&self) -> MessageSink {
+		self.get::<Display>(ObjectId::DISPLAY)
+			.unwrap()
+			.message_sink
+			.clone()
+	}
+
+	fn display(&self) -> Arc<Display> {
+		self.get::<Display>(ObjectId::DISPLAY).unwrap()
+	}
+
+	async fn protocol_error(
+		&mut self,
+		sender_id: ObjectId,
+		object_id: ObjectId,
+		code: u32,
+		message: String,
+	) -> Result<()> {
+		self.display()
+			.error(self, sender_id, object_id, code, message)
+			.await?;
+		let _ = self.message_sink().send(Message::Disconnect);
+
+		Ok(())
+	}
+}
+
+#[derive(Debug, Default)]
+pub struct DoubleBuffer<State: Debug + Clone> {
+	pub current: State,
+	pub pending: State,
+}
+impl<State: Debug + Clone> DoubleBuffer<State> {
+	pub fn new(initial_state: State) -> Self {
+		DoubleBuffer {
+			current: initial_state.clone(),
+			pending: initial_state,
+		}
+	}
+	pub fn apply(&mut self) {
+		self.current = self.pending.clone();
+	}
+	pub fn current(&self) -> &State {
+		&self.current
+	}
+}

src/wayland/utils.rs

@@ -1,121 +0,0 @@
-use mint::Vector2;
-use parking_lot::Mutex;
-use smithay::{
-	backend::renderer::utils::RendererSurfaceStateUserData,
-	reexports::wayland_server::protocol::wl_surface::WlSurface,
-	wayland::{
-		compositor,
-		shell::xdg::{SurfaceCachedState, XdgToplevelSurfaceData},
-	},
-};
-
-use crate::nodes::items::panel::{ChildInfo, Geometry, ToplevelInfo};
-
-use super::xdg_shell::surface_panel_item;
-pub trait WlSurfaceExt {
-	fn insert_data<T: Send + Sync + 'static>(&self, data: T) -> bool;
-	fn get_data<T: Send + Sync + Clone + 'static>(&self) -> Option<T>;
-	fn get_data_raw<T: Send + Sync + 'static, O, F: FnOnce(&T) -> O>(&self, f: F) -> Option<O>;
-	fn get_current_surface_state(&self) -> SurfaceCachedState;
-	fn get_pending_surface_state(&self) -> SurfaceCachedState;
-	fn get_size(&self) -> Option<Vector2<u32>>;
-	fn get_geometry(&self) -> Option<Geometry>;
-}
-impl WlSurfaceExt for WlSurface {
-	fn insert_data<T: Send + Sync + 'static>(&self, data: T) -> bool {
-		compositor::with_states(self, |d| {
-			d.data_map.insert_if_missing_threadsafe(move || data)
-		})
-	}
-	fn get_data<T: Send + Sync + Clone + 'static>(&self) -> Option<T> {
-		compositor::with_states(self, |d| d.data_map.get::<T>().cloned())
-	}
-	fn get_data_raw<T: Send + Sync + 'static, O, F: FnOnce(&T) -> O>(&self, f: F) -> Option<O> {
-		compositor::with_states(self, |d| Some((f)(d.data_map.get::<T>()?)))
-	}
-	fn get_current_surface_state(&self) -> SurfaceCachedState {
-		compositor::with_states(self, |states| {
-			*states.cached_state.get::<SurfaceCachedState>().current()
-		})
-	}
-	fn get_pending_surface_state(&self) -> SurfaceCachedState {
-		compositor::with_states(self, |states| {
-			*states.cached_state.get::<SurfaceCachedState>().pending()
-		})
-	}
-	fn get_size(&self) -> Option<Vector2<u32>> {
-		self.get_data_raw::<RendererSurfaceStateUserData, _, _>(|surface_states| {
-			surface_states.lock().unwrap().surface_size()
-		})
-		.flatten()
-		.map(|size| Vector2::from([size.w as u32, size.h as u32]))
-	}
-	fn get_geometry(&self) -> Option<Geometry> {
-		self.get_current_surface_state().geometry.map(|r| r.into())
-	}
-}
-
-pub trait ToplevelInfoExt {
-	fn get_toplevel_info(&self) -> Option<ToplevelInfo>;
-	fn with_toplevel_info<O, F: FnOnce(&mut ToplevelInfo) -> O>(&self, f: F) -> Option<O>;
-
-	fn get_parent(&self) -> Option<u64>;
-	fn get_app_id(&self) -> Option<String>;
-	fn get_title(&self) -> Option<String>;
-	fn min_size(&self) -> Option<Vector2<u32>>;
-	fn max_size(&self) -> Option<Vector2<u32>>;
-}
-impl ToplevelInfoExt for WlSurface {
-	fn get_toplevel_info(&self) -> Option<ToplevelInfo> {
-		self.get_data_raw::<Mutex<ToplevelInfo>, _, _>(|c| c.lock().clone())
-	}
-	fn with_toplevel_info<O, F: FnOnce(&mut ToplevelInfo) -> O>(&self, f: F) -> Option<O> {
-		self.get_data_raw::<Mutex<ToplevelInfo>, _, _>(|r| (f)(&mut r.lock()))
-	}
-
-	fn get_parent(&self) -> Option<u64> {
-		self.get_data_raw::<XdgToplevelSurfaceData, _, _>(|d| d.lock().unwrap().parent.clone())
-			.flatten()
-			.and_then(|p| surface_panel_item(&p))
-			.and_then(|p| p.node.upgrade())
-			.map(|p| p.get_id())
-	}
-	fn get_app_id(&self) -> Option<String> {
-		self.get_data_raw::<XdgToplevelSurfaceData, _, _>(|d| d.lock().ok()?.app_id.clone())
-			.flatten()
-	}
-	fn get_title(&self) -> Option<String> {
-		self.get_data_raw::<XdgToplevelSurfaceData, _, _>(|d| d.lock().ok()?.title.clone())
-			.flatten()
-	}
-	fn min_size(&self) -> Option<Vector2<u32>> {
-		let state = self.get_pending_surface_state();
-		let size = state.min_size;
-		if size.w == 0 && size.h == 0 {
-			None
-		} else {
-			Some(Vector2::from([size.w as u32, size.h as u32]))
-		}
-	}
-	fn max_size(&self) -> Option<Vector2<u32>> {
-		let state = self.get_pending_surface_state();
-		let size = state.max_size;
-		if size.w == 0 && size.h == 0 {
-			None
-		} else {
-			Some(Vector2::from([size.w as u32, size.h as u32]))
-		}
-	}
-}
-pub trait ChildInfoExt {
-	fn get_child_info(&self) -> Option<ChildInfo>;
-	fn with_child_info<O, F: FnOnce(&mut ChildInfo) -> O>(&self, f: F) -> Option<O>;
-}
-impl ChildInfoExt for WlSurface {
-	fn get_child_info(&self) -> Option<ChildInfo> {
-		self.get_data_raw::<Mutex<ChildInfo>, _, _>(|c| c.lock().clone())
-	}
-	fn with_child_info<O, F: FnOnce(&mut ChildInfo) -> O>(&self, f: F) -> Option<O> {
-		self.get_data_raw::<Mutex<ChildInfo>, _, _>(|r| (f)(&mut r.lock()))
-	}
-}

src/wayland/vulkano_data.rs

@@ -0,0 +1,145 @@
+use std::sync::{Arc, OnceLock};
+
+use bevy::{
+	ecs::system::Res,
+	render::renderer::{RenderAdapter, RenderDevice, RenderInstance},
+};
+use vulkano::{
+	VulkanLibrary,
+	command_buffer::allocator::{
+		CommandBufferAllocator, StandardCommandBufferAllocator,
+		StandardCommandBufferAllocatorCreateInfo,
+	},
+	device::{DeviceCreateInfo, QueueCreateInfo},
+	instance::InstanceCreateFlags,
+	memory::allocator::{MemoryAllocator, StandardMemoryAllocator},
+};
+use wgpu_hal::vulkan::Api as VulkanHal;
+
+pub static VULKANO_CONTEXT: OnceLock<VulkanoContext> = OnceLock::new();
+
+#[expect(dead_code)]
+pub struct VulkanoContext {
+	pub instance: Arc<vulkano::instance::Instance>,
+	pub phys_dev: Arc<vulkano::device::physical::PhysicalDevice>,
+	pub dev: Arc<vulkano::device::Device>,
+	pub queue: Arc<vulkano::device::Queue>,
+	pub alloc: Arc<dyn MemoryAllocator>,
+	pub command_buffer_alloc: Arc<dyn CommandBufferAllocator>,
+}
+
+pub fn setup_vulkano_context(
+	dev: Res<RenderDevice>,
+	instance: Res<RenderInstance>,
+	adapter: Res<RenderAdapter>,
+) {
+	if VULKANO_CONTEXT.get().is_some() {
+		return;
+	}
+	let hal_instance = unsafe { instance.as_hal::<VulkanHal>() }
+		.unwrap()
+		.shared_instance();
+
+	let ash_instance = hal_instance.raw_instance();
+	let vulkan_lib =
+		VulkanLibrary::with_loader(AshEntryVulkanoLoader(hal_instance.entry().clone())).unwrap();
+
+	let vulkano_instance = unsafe {
+		vulkano::instance::Instance::from_handle(
+			vulkan_lib,
+			ash_instance.handle(),
+			vulkano::instance::InstanceCreateInfo {
+				flags: InstanceCreateFlags::empty(),
+				// TODO: make vulkan init reasonable and remove this hardcoded value from
+				// bevy_mod_openxr
+				max_api_version: Some(vulkano::Version::V1_2),
+				enabled_extensions: vulkano::instance::InstanceExtensions::from_iter(
+					hal_instance
+						.extensions()
+						.iter()
+						.map(|s| s.to_str().unwrap()),
+				),
+				..Default::default()
+			},
+		)
+	};
+
+	let ash_phys_dev_handle = unsafe {
+		adapter.as_hal::<VulkanHal, _, _>(|adapter| adapter.unwrap().raw_physical_device())
+	};
+
+	let vulkano_phys_dev = unsafe {
+		vulkano::device::physical::PhysicalDevice::from_handle(
+			vulkano_instance.clone(),
+			ash_phys_dev_handle,
+		)
+	}
+	.unwrap();
+	let (ash_dev_handle, dev_create_info) = unsafe {
+		dev.wgpu_device().as_hal::<VulkanHal, _, _>(|dev| {
+			let dev = dev.unwrap();
+			(
+				dev.raw_device().handle(),
+				DeviceCreateInfo {
+					queue_create_infos: vec![QueueCreateInfo {
+						queue_family_index: dev.queue_family_index(),
+						..Default::default()
+					}],
+					enabled_extensions: vulkano::device::DeviceExtensions::from_iter(
+						dev.enabled_device_extensions()
+							.iter()
+							// TODO: remove this hack by telling wgpu about the actual exts used in
+							// bevy_mod_openxr
+							.chain(bevy_dmabuf::required_device_extensions().iter())
+							.map(|v| v.to_str().unwrap()),
+					),
+					// this is def wrong, lets hope it doesn't cause issues....
+					enabled_features: vulkano::device::DeviceFeatures::empty(),
+
+					..Default::default()
+				},
+			)
+		})
+	};
+	let (vulkano_dev, mut queues) = unsafe {
+		vulkano::device::Device::from_handle(
+			vulkano_phys_dev.clone(),
+			ash_dev_handle,
+			dev_create_info,
+		)
+	};
+
+	let alloc = Arc::new(StandardMemoryAllocator::new_default(vulkano_dev.clone()));
+
+	let command_buffer_alloc = Arc::new(StandardCommandBufferAllocator::new(
+		vulkano_dev.clone(),
+		StandardCommandBufferAllocatorCreateInfo::default(),
+	));
+	_ = VULKANO_CONTEXT.set(VulkanoContext {
+		instance: vulkano_instance,
+		phys_dev: vulkano_phys_dev,
+		dev: vulkano_dev,
+		queue: queues.next().unwrap(),
+		alloc,
+		command_buffer_alloc,
+	});
+}
+
+// ensures that we don't destroy the vulkan handles wgpu/bevy use
+// TODO: remove once we don't use bevys wgpu instance/device/physical_device
+impl Drop for VulkanoContext {
+	fn drop(&mut self) {
+		panic!("the vulkano context shall never be dropped");
+	}
+}
+
+struct AshEntryVulkanoLoader(ash::Entry);
+unsafe impl vulkano::library::Loader for AshEntryVulkanoLoader {
+	unsafe fn get_instance_proc_addr(
+		&self,
+		instance: ash::vk::Instance,
+		name: *const std::os::raw::c_char,
+	) -> ash::vk::PFN_vkVoidFunction {
+		unsafe { self.0.get_instance_proc_addr(instance, name) }
+	}
+}

src/wayland/wayland-drm.xml

@@ -1,189 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<protocol name="drm">
-
-    <copyright>
-        Copyright © 2008-2011 Kristian Høgsberg
-        Copyright © 2010-2011 Intel Corporation
-
-        Permission to use, copy, modify, distribute, and sell this
-        software and its documentation for any purpose is hereby granted
-        without fee, provided that\n the above copyright notice appear in
-        all copies and that both that copyright notice and this permission
-        notice appear in supporting documentation, and that the name of
-        the copyright holders not be used in advertising or publicity
-        pertaining to distribution of the software without specific,
-        written prior permission. The copyright holders make no
-        representations about the suitability of this software for any
-        purpose. It is provided "as is" without express or implied
-        warranty.
-
-        THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS
-        SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
-        FITNESS, IN NO EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY
-        SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
-        WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
-        AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION,
-        ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF
-        THIS SOFTWARE.
-    </copyright>
-
-    <!-- drm support. This object is created by the server and published
-       using the display's global event. -->
-    <interface name="wl_drm" version="2">
-        <enum name="error">
-            <entry name="authenticate_fail" value="0" />
-            <entry name="invalid_format" value="1" />
-            <entry name="invalid_name" value="2" />
-        </enum>
-
-        <enum name="format">
-            <!-- The drm format codes match the #defines in drm_fourcc.h.
-           The formats actually supported by the compositor will be
-           reported by the format event. New codes must not be added,
-           unless directly taken from drm_fourcc.h. -->
-            <entry name="c8" value="0x20203843" />
-            <entry name="rgb332" value="0x38424752" />
-            <entry name="bgr233" value="0x38524742" />
-            <entry name="xrgb4444" value="0x32315258" />
-            <entry name="xbgr4444" value="0x32314258" />
-            <entry name="rgbx4444" value="0x32315852" />
-            <entry name="bgrx4444" value="0x32315842" />
-            <entry name="argb4444" value="0x32315241" />
-            <entry name="abgr4444" value="0x32314241" />
-            <entry name="rgba4444" value="0x32314152" />
-            <entry name="bgra4444" value="0x32314142" />
-            <entry name="xrgb1555" value="0x35315258" />
-            <entry name="xbgr1555" value="0x35314258" />
-            <entry name="rgbx5551" value="0x35315852" />
-            <entry name="bgrx5551" value="0x35315842" />
-            <entry name="argb1555" value="0x35315241" />
-            <entry name="abgr1555" value="0x35314241" />
-            <entry name="rgba5551" value="0x35314152" />
-            <entry name="bgra5551" value="0x35314142" />
-            <entry name="rgb565" value="0x36314752" />
-            <entry name="bgr565" value="0x36314742" />
-            <entry name="rgb888" value="0x34324752" />
-            <entry name="bgr888" value="0x34324742" />
-            <entry name="xrgb8888" value="0x34325258" />
-            <entry name="xbgr8888" value="0x34324258" />
-            <entry name="rgbx8888" value="0x34325852" />
-            <entry name="bgrx8888" value="0x34325842" />
-            <entry name="argb8888" value="0x34325241" />
-            <entry name="abgr8888" value="0x34324241" />
-            <entry name="rgba8888" value="0x34324152" />
-            <entry name="bgra8888" value="0x34324142" />
-            <entry name="xrgb2101010" value="0x30335258" />
-            <entry name="xbgr2101010" value="0x30334258" />
-            <entry name="rgbx1010102" value="0x30335852" />
-            <entry name="bgrx1010102" value="0x30335842" />
-            <entry name="argb2101010" value="0x30335241" />
-            <entry name="abgr2101010" value="0x30334241" />
-            <entry name="rgba1010102" value="0x30334152" />
-            <entry name="bgra1010102" value="0x30334142" />
-            <entry name="yuyv" value="0x56595559" />
-            <entry name="yvyu" value="0x55595659" />
-            <entry name="uyvy" value="0x59565955" />
-            <entry name="vyuy" value="0x59555956" />
-            <entry name="ayuv" value="0x56555941" />
-            <entry name="xyuv8888" value="0x56555958" />
-            <entry name="nv12" value="0x3231564e" />
-            <entry name="nv21" value="0x3132564e" />
-            <entry name="nv16" value="0x3631564e" />
-            <entry name="nv61" value="0x3136564e" />
-            <entry name="yuv410" value="0x39565559" />
-            <entry name="yvu410" value="0x39555659" />
-            <entry name="yuv411" value="0x31315559" />
-            <entry name="yvu411" value="0x31315659" />
-            <entry name="yuv420" value="0x32315559" />
-            <entry name="yvu420" value="0x32315659" />
-            <entry name="yuv422" value="0x36315559" />
-            <entry name="yvu422" value="0x36315659" />
-            <entry name="yuv444" value="0x34325559" />
-            <entry name="yvu444" value="0x34325659" />
-            <entry name="abgr16f" value="0x48344241" />
-            <entry name="xbgr16f" value="0x48344258" />
-        </enum>
-
-        <!-- Call this request with the magic received from drmGetMagic().
-         It will be passed on to the drmAuthMagic() or
-         DRIAuthConnection() call.  This authentication must be
-         completed before create_buffer could be used. -->
-        <request name="authenticate">
-            <arg name="id" type="uint" />
-        </request>
-
-        <!-- Create a wayland buffer for the named DRM buffer.  The DRM
-         surface must have a name using the flink ioctl -->
-        <request name="create_buffer">
-            <arg name="id" type="new_id" interface="wl_buffer" />
-            <arg name="name" type="uint" />
-            <arg name="width" type="int" />
-            <arg name="height" type="int" />
-            <arg name="stride" type="uint" />
-            <arg name="format" type="uint" />
-        </request>
-
-        <!-- Create a wayland buffer for the named DRM buffer.  The DRM
-         surface must have a name using the flink ioctl -->
-        <request name="create_planar_buffer">
-            <arg name="id" type="new_id" interface="wl_buffer" />
-            <arg name="name" type="uint" />
-            <arg name="width" type="int" />
-            <arg name="height" type="int" />
-            <arg name="format" type="uint" />
-            <arg name="offset0" type="int" />
-            <arg name="stride0" type="int" />
-            <arg name="offset1" type="int" />
-            <arg name="stride1" type="int" />
-            <arg name="offset2" type="int" />
-            <arg name="stride2" type="int" />
-        </request>
-
-        <!-- Notification of the path of the drm device which is used by
-         the server.  The client should use this device for creating
-         local buffers.  Only buffers created from this device should
-         be be passed to the server using this drm object's
-         create_buffer request. -->
-        <event name="device">
-            <arg name="name" type="string" />
-        </event>
-
-        <event name="format">
-            <arg name="format" type="uint" />
-        </event>
-
-        <!-- Raised if the authenticate request succeeded -->
-        <event name="authenticated" />
-
-        <enum name="capability" since="2">
-            <description summary="wl_drm capability bitmask">
-                Bitmask of capabilities.
-            </description>
-            <entry name="prime" value="1" summary="wl_drm prime available" />
-        </enum>
-
-        <event name="capabilities">
-            <arg name="value" type="uint" />
-        </event>
-
-        <!-- Version 2 additions -->
-
-        <!-- Create a wayland buffer for the prime fd.  Use for regular and planar
-         buffers.  Pass 0 for offset and stride for unused planes. -->
-        <request name="create_prime_buffer" since="2">
-            <arg name="id" type="new_id" interface="wl_buffer" />
-            <arg name="name" type="fd" />
-            <arg name="width" type="int" />
-            <arg name="height" type="int" />
-            <arg name="format" type="uint" />
-            <arg name="offset0" type="int" />
-            <arg name="stride0" type="int" />
-            <arg name="offset1" type="int" />
-            <arg name="stride1" type="int" />
-            <arg name="offset2" type="int" />
-            <arg name="stride2" type="int" />
-        </request>
-
-    </interface>
-
-</protocol>

src/wayland/xdg/backend.rs

@@ -0,0 +1,236 @@
+use super::toplevel::Toplevel;
+use crate::{
+        core::error::Result,
+        nodes::{
+                drawable::model::ModelPart,
+                items::panel::{Backend, Geometry, PanelItemInitData, SurfaceId, ToplevelInfo},
+        },
+        wayland::{Message, core::surface::Surface},
+};
+use mint::Vector2;
+use parking_lot::Mutex;
+use std::{collections::HashMap, sync::{Arc, Weak}};
+use tracing;
+
+#[derive(Debug)]
+pub struct XdgBackend {
+        toplevel: Weak<Toplevel>,
+        children: Mutex<HashMap<u64, Weak<Surface>>>,
+}
+
+impl XdgBackend {
+        pub fn new(toplevel: Arc<Toplevel>) -> Self {
+                Self {
+                        toplevel: Arc::downgrade(&toplevel),
+                        children: Mutex::new(HashMap::new()),
+                }
+        }
+
+	// Since XdgBackend is created and owned by Mapped which is owned by Toplevel,
+	// we can safely assume the Toplevel reference will always be valid
+	fn toplevel(&self) -> Arc<Toplevel> {
+		self.toplevel
+			.upgrade()
+			.expect("Toplevel should always be valid while XdgBackend exists")
+	}
+
+        fn surface_from_id(&self, id: SurfaceId) -> Option<Arc<Surface>> {
+                match id {
+                        SurfaceId::Toplevel(_) => Some(self.toplevel().surface()),
+                        SurfaceId::Child(uid) => self.children.lock().get(&uid).and_then(Weak::upgrade),
+                }
+        }
+
+        pub fn register_child(&self, id: u64, surface: &Arc<Surface>) {
+                self.children.lock().insert(id, Arc::downgrade(surface));
+        }
+
+        pub fn unregister_child(&self, id: u64) {
+                self.children.lock().remove(&id);
+        }
+}
+
+impl Backend for XdgBackend {
+	fn start_data(&self) -> Result<PanelItemInitData> {
+		let surface_state = self.toplevel().surface().current_state();
+
+		let size = surface_state
+			.buffer
+			.map(|b| [b.buffer.size().x as u32, b.buffer.size().y as u32].into())
+			.unwrap_or([0; 2].into());
+		let toplevel = ToplevelInfo {
+			parent: self.toplevel().parent(),
+			title: self.toplevel().title(),
+			app_id: self.toplevel().app_id(),
+			size,
+			min_size: surface_state
+				.min_size
+				.map(|v| [v.x as f32, v.y as f32].into()),
+			max_size: surface_state
+				.max_size
+				.map(|v| [v.x as f32, v.y as f32].into()),
+			logical_rectangle: surface_state.geometry.unwrap_or(Geometry {
+				origin: [0; 2].into(),
+				size,
+			}),
+		};
+
+		Ok(PanelItemInitData {
+			cursor: None,
+			toplevel,
+			children: vec![],
+			pointer_grab: None,
+			keyboard_grab: None,
+		})
+	}
+
+	fn apply_cursor_material(&self, _model_part: &Arc<ModelPart>) {}
+	fn apply_surface_material(&self, surface: SurfaceId, model_part: &Arc<ModelPart>) {
+		if let Some(surface) = self.surface_from_id(surface) {
+			surface.apply_material(model_part);
+		}
+	}
+
+	fn close_toplevel(&self) {
+		let _ = self
+			.toplevel()
+			.surface()
+			.message_sink
+			.send(Message::CloseToplevel(self.toplevel().clone()));
+	}
+
+	fn auto_size_toplevel(&self) {
+		let _ = self
+			.toplevel()
+			.surface()
+			.message_sink
+			.send(Message::ResizeToplevel {
+				toplevel: self.toplevel().clone(),
+				size: None,
+			});
+	}
+
+	fn set_toplevel_size(&self, size: Vector2<u32>) {
+		let _ = self
+			.toplevel()
+			.surface()
+			.message_sink
+			.send(Message::ResizeToplevel {
+				toplevel: self.toplevel().clone(),
+				size: Some(size),
+			});
+	}
+
+	fn set_toplevel_focused_visuals(&self, focused: bool) {
+		let _ = self
+			.toplevel()
+			.surface()
+			.message_sink
+			.send(Message::SetToplevelVisualActive {
+				toplevel: self.toplevel().clone(),
+				active: focused,
+			});
+	}
+
+	fn pointer_motion(&self, surface: &SurfaceId, position: Vector2<f32>) {
+		if let Some(surface) = self.surface_from_id(surface.clone()) {
+			let _ = self.toplevel().surface().message_sink.send(Message::Seat(
+				crate::wayland::core::seat::SeatMessage::PointerMotion { surface, position },
+			));
+		}
+	}
+
+	fn pointer_button(&self, surface: &SurfaceId, button: u32, pressed: bool) {
+		if let Some(surface) = self.surface_from_id(surface.clone()) {
+			let _ = self.toplevel().surface().message_sink.send(Message::Seat(
+				crate::wayland::core::seat::SeatMessage::PointerButton {
+					surface,
+					button,
+					pressed,
+				},
+			));
+		}
+	}
+
+	fn pointer_scroll(
+		&self,
+		surface: &SurfaceId,
+		scroll_distance: Option<Vector2<f32>>,
+		scroll_steps: Option<Vector2<f32>>,
+	) {
+		if let Some(surface) = self.surface_from_id(surface.clone()) {
+			let _ = self.toplevel().surface().message_sink.send(Message::Seat(
+				crate::wayland::core::seat::SeatMessage::PointerScroll {
+					surface,
+					scroll_distance,
+					scroll_steps,
+				},
+			));
+		}
+	}
+
+	fn keyboard_key(&self, surface: &SurfaceId, keymap_id: u64, key: u32, pressed: bool) {
+		tracing::debug!(
+			"Backend: Keyboard key {} {}",
+			key,
+			if pressed { "pressed" } else { "released" }
+		);
+		if let Some(surface) = self.surface_from_id(surface.clone()) {
+			let _ = self.toplevel().surface().message_sink.send(Message::Seat(
+				crate::wayland::core::seat::SeatMessage::KeyboardKey {
+					surface,
+					keymap_id,
+					key,
+					pressed,
+				},
+			));
+		}
+	}
+
+	fn touch_down(&self, surface: &SurfaceId, id: u32, position: Vector2<f32>) {
+		tracing::debug!(
+			"Backend: Touch down {} at ({}, {})",
+			id,
+			position.x,
+			position.y
+		);
+		if let Some(surface) = self.surface_from_id(surface.clone()) {
+			let _ = self.toplevel().surface().message_sink.send(Message::Seat(
+				crate::wayland::core::seat::SeatMessage::TouchDown {
+					surface,
+					id,
+					position,
+				},
+			));
+		}
+	}
+
+	fn touch_move(&self, id: u32, position: Vector2<f32>) {
+		tracing::debug!(
+			"Backend: Touch move {} to ({}, {})",
+			id,
+			position.x,
+			position.y
+		);
+		let surface = self.toplevel().surface();
+		let _ = surface.message_sink.send(Message::Seat(
+			crate::wayland::core::seat::SeatMessage::TouchMove { id, position },
+		));
+	}
+
+	fn touch_up(&self, id: u32) {
+		tracing::debug!("Backend: Touch up {}", id);
+		let surface = self.toplevel().surface();
+		let _ = surface.message_sink.send(Message::Seat(
+			crate::wayland::core::seat::SeatMessage::TouchUp { id },
+		));
+	}
+
+	fn reset_input(&self) {
+		tracing::debug!("Backend: Reset input");
+		let surface = self.toplevel().surface();
+		let _ = surface.message_sink.send(Message::Seat(
+			crate::wayland::core::seat::SeatMessage::Reset,
+		));
+	}
+}

src/wayland/xdg/mod.rs

@@ -0,0 +1,6 @@
+pub mod backend;
+pub mod popup;
+pub mod positioner;
+pub mod surface;
+pub mod toplevel;
+pub mod wm_base;

src/wayland/xdg/popup.rs

@@ -0,0 +1,178 @@
+use super::{
+	backend::XdgBackend,
+	positioner::{Positioner, PositionerData},
+	surface::Surface,
+};
+use crate::{
+        nodes::items::panel::{ChildInfo, Geometry, PanelItem, SurfaceId},
+        wayland::util::DoubleBuffer,
+};
+use parking_lot::Mutex;
+use rand::Rng;
+use std::{
+	sync::{Arc, Weak, atomic::AtomicBool},
+	u64,
+};
+use waynest::{
+	server::{Client, Dispatcher, Result, protocol::stable::xdg_shell::xdg_popup::XdgPopup},
+	wire::ObjectId,
+};
+
+#[derive(Debug, Dispatcher)]
+pub struct Popup {
+	id: ObjectId,
+	version: u32,
+	surface_id: SurfaceId,
+	parent: Arc<Surface>,
+	surface: Weak<Surface>,
+        pub panel_item: Weak<PanelItem<XdgBackend>>,
+        positioner_data: Mutex<PositionerData>,
+        geometry: Mutex<DoubleBuffer<Geometry>>,
+        mapped: AtomicBool,
+}
+impl Popup {
+	pub fn new(
+		id: ObjectId,
+		version: u32,
+		parent: Arc<Surface>,
+		panel_item: &Arc<PanelItem<XdgBackend>>,
+		xdg_surface: &Arc<Surface>,
+		positioner: &Positioner,
+	) -> Self {
+		let positioner_data = positioner.data();
+		Self {
+			id,
+			version,
+			surface_id: SurfaceId::Child(rand::thread_rng().gen_range(0..u64::MAX)),
+			parent,
+			surface: Arc::downgrade(xdg_surface),
+                        panel_item: Arc::downgrade(panel_item),
+                        positioner_data: Mutex::new(positioner_data),
+                        geometry: Mutex::new(DoubleBuffer::new(positioner_data.infinite_geometry())),
+                        mapped: AtomicBool::new(false),
+                }
+        }
+}
+
+impl Popup {
+        fn id(&self) -> u64 {
+                match self.surface_id {
+                        SurfaceId::Child(id) => id,
+                        SurfaceId::Toplevel(_) => 0,
+                }
+        }
+
+        pub fn surface_id(&self) -> SurfaceId { self.surface_id.clone() }
+
+        pub fn current_geometry(&self) -> Geometry {
+                *self.geometry.lock().current()
+        }
+
+        pub fn is_mapped(&self) -> bool {
+                self.mapped.load(std::sync::atomic::Ordering::SeqCst)
+        }
+
+        pub fn map(&self) {
+                if self.is_mapped() {
+                        return;
+                }
+
+                let Some(panel_item) = self.panel_item.upgrade() else { return; };
+                let xdg_surface = match self.surface.upgrade() { Some(s) => s, None => return };
+                let core_surface = xdg_surface.wl_surface();
+
+                // Determine parent surface id
+                let parent_wl_surface = self.parent.wl_surface();
+                let parent_role = parent_wl_surface.role.lock();
+                let parent_id = match parent_role.as_ref().unwrap() {
+                        crate::wayland::core::surface::SurfaceRole::XdgToplevel(_) => {
+                                SurfaceId::Toplevel(())
+                        }
+                        crate::wayland::core::surface::SurfaceRole::XDGPopup(p) => p.surface_id(),
+                };
+
+                let geometry = *self.geometry.lock().current();
+                let info = ChildInfo {
+                        id: self.id(),
+                        parent: parent_id,
+                        geometry,
+                        z_order: 0,
+                        receives_input: true,
+                };
+                panel_item.create_child(self.id(), &info);
+                panel_item.backend.register_child(self.id(), &core_surface);
+                self.mapped.store(true, std::sync::atomic::Ordering::SeqCst);
+        }
+
+        pub fn unmap(&self) {
+                if !self.mapped.swap(false, std::sync::atomic::Ordering::SeqCst) {
+                        return;
+                }
+
+                if let Some(panel_item) = self.panel_item.upgrade() {
+                        panel_item.destroy_child(self.id());
+                        panel_item.backend.unregister_child(self.id());
+                }
+        }
+
+        fn reposition_child(&self) {
+                if self.is_mapped() {
+                        if let Some(panel_item) = self.panel_item.upgrade() {
+                                let geometry = *self.geometry.lock().current();
+                                panel_item.reposition_child(self.id(), &geometry);
+                        }
+                }
+        }
+}
+impl XdgPopup for Popup {
+	/// https://wayland.app/protocols/xdg-shell#xdg_popup:request:grab
+	async fn grab(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_seat: ObjectId,
+		_serial: u32,
+	) -> Result<()> {
+		Ok(())
+	}
+
+	/// https://wayland.app/protocols/xdg-shell#xdg_popup:request:reposition
+	async fn reposition(
+		&self,
+		client: &mut Client,
+		sender_id: ObjectId,
+		positioner: ObjectId,
+		token: u32,
+	) -> Result<()> {
+                let positioner = client.get::<Positioner>(positioner).unwrap();
+                let positioner_data = positioner.data();
+                *self.positioner_data.lock() = positioner_data;
+                if self.version >= 5 {
+                        self.repositioned(client, sender_id, token).await?;
+                }
+                let geometry = positioner_data.infinite_geometry();
+                {
+                        let mut geo = self.geometry.lock();
+                        geo.pending = geometry;
+                        geo.apply();
+                }
+                self.configure(
+                        client,
+                        sender_id,
+                        geometry.origin.x,
+                        geometry.origin.y,
+                        geometry.size.x as i32,
+                        geometry.size.y as i32,
+                )
+                .await?;
+                self.surface.upgrade().unwrap().reconfigure(client).await?;
+                self.reposition_child();
+                Ok(())
+        }
+
+	/// https://wayland.app/protocols/xdg-shell#xdg_popup:request:destroy
+        async fn destroy(&self, _client: &mut Client, _sender_id: ObjectId) -> Result<()> {
+                self.unmap();
+                Ok(())
+        }
+}

src/wayland/xdg/positioner.rs

@@ -0,0 +1,229 @@
+use crate::nodes::items::panel::Geometry;
+use mint::Vector2;
+use parking_lot::Mutex;
+use waynest::{
+	server::{
+		Client, Dispatcher, Result,
+		protocol::stable::xdg_shell::xdg_positioner::{
+			Anchor, ConstraintAdjustment, Gravity, XdgPositioner,
+		},
+	},
+	wire::ObjectId,
+};
+
+#[derive(Debug, Clone, Copy)]
+pub struct PositionerData {
+	pub size: Vector2<u32>,
+	pub anchor_rect: Geometry,
+	pub offset: Vector2<i32>,
+	pub anchor: Anchor,
+	pub constraint_adjustment: ConstraintAdjustment,
+	pub reactive: bool,
+	pub parent_size: Vector2<u32>,
+}
+impl PositionerData {
+	pub fn infinite_geometry(&self) -> Geometry {
+		let anchor_point = match self.anchor {
+			Anchor::TopLeft => self.anchor_rect.origin,
+			Anchor::Top => [
+				self.anchor_rect.origin.x + (self.anchor_rect.size.x / 2) as i32,
+				self.anchor_rect.origin.y,
+			]
+			.into(),
+			Anchor::TopRight => [
+				self.anchor_rect.origin.x + self.anchor_rect.size.x as i32,
+				self.anchor_rect.origin.y,
+			]
+			.into(),
+			Anchor::Left => [
+				self.anchor_rect.origin.x,
+				self.anchor_rect.origin.y + (self.anchor_rect.size.y / 2) as i32,
+			]
+			.into(),
+			Anchor::Right => [
+				self.anchor_rect.origin.x + self.anchor_rect.size.x as i32,
+				self.anchor_rect.origin.y + (self.anchor_rect.size.y / 2) as i32,
+			]
+			.into(),
+			Anchor::BottomLeft => [
+				self.anchor_rect.origin.x,
+				self.anchor_rect.origin.y + self.anchor_rect.size.y as i32,
+			]
+			.into(),
+			Anchor::Bottom => [
+				self.anchor_rect.origin.x + (self.anchor_rect.size.x / 2) as i32,
+				self.anchor_rect.origin.y + self.anchor_rect.size.y as i32,
+			]
+			.into(),
+			Anchor::BottomRight => [
+				self.anchor_rect.origin.x + self.anchor_rect.size.x as i32,
+				self.anchor_rect.origin.y + self.anchor_rect.size.y as i32,
+			]
+			.into(),
+			_ => [
+				self.anchor_rect.origin.x + (self.anchor_rect.size.x / 2) as i32,
+				self.anchor_rect.origin.y + (self.anchor_rect.size.y / 2) as i32,
+			]
+			.into(),
+		};
+
+		let mut position = anchor_point;
+
+		// Apply gravity
+		if self
+			.constraint_adjustment
+			.contains(ConstraintAdjustment::FlipX)
+		{
+			position.x -= self.size.x as i32;
+		}
+		if self
+			.constraint_adjustment
+			.contains(ConstraintAdjustment::FlipY)
+		{
+			position.y -= self.size.y as i32;
+		}
+
+		// Apply offset
+		position.x += self.offset.x;
+		position.y += self.offset.y;
+
+		Geometry {
+			origin: position,
+			size: self.size,
+		}
+	}
+}
+impl Default for PositionerData {
+	fn default() -> Self {
+		Self {
+			size: [0; 2].into(),
+			anchor_rect: Default::default(),
+			offset: [0, 0].into(),
+			anchor: Anchor::TopLeft,
+			constraint_adjustment: ConstraintAdjustment::empty(),
+			reactive: false,
+			parent_size: [0; 2].into(),
+		}
+	}
+}
+
+#[derive(Debug, Dispatcher)]
+pub struct Positioner {
+	data: Mutex<PositionerData>,
+}
+impl Default for Positioner {
+	fn default() -> Self {
+		Self {
+			data: Mutex::new(PositionerData::default()),
+		}
+	}
+}
+impl Positioner {
+	pub fn data(&self) -> PositionerData {
+		*self.data.lock()
+	}
+}
+impl XdgPositioner for Positioner {
+	async fn set_size(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_width: i32,
+		_height: i32,
+	) -> Result<()> {
+		let mut data = self.data.lock();
+		data.size = [_width.max(0) as u32, _height.max(0) as u32].into();
+		data.reactive = true;
+		Ok(())
+	}
+
+	async fn set_anchor_rect(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_x: i32,
+		_y: i32,
+		_width: i32,
+		_height: i32,
+	) -> Result<()> {
+		let mut data = self.data.lock();
+		data.anchor_rect.origin = [_x, _y].into();
+		data.anchor_rect.size = [_width.max(0) as u32, _height.max(0) as u32].into();
+		data.offset = [0, 0].into();
+		Ok(())
+	}
+
+	async fn set_anchor(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_anchor: Anchor,
+	) -> Result<()> {
+		let mut data = self.data.lock();
+		data.anchor = _anchor;
+		Ok(())
+	}
+
+	async fn set_gravity(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_gravity: Gravity,
+	) -> Result<()> {
+		Ok(())
+	}
+
+	async fn set_constraint_adjustment(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_constraint_adjustment: ConstraintAdjustment,
+	) -> Result<()> {
+		let mut data = self.data.lock();
+		data.constraint_adjustment = _constraint_adjustment;
+		Ok(())
+	}
+
+	async fn set_offset(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_x: i32,
+		_y: i32,
+	) -> Result<()> {
+		let mut data = self.data.lock();
+		data.offset.x += _x;
+		data.offset.y += _y;
+		Ok(())
+	}
+
+	async fn set_reactive(&self, _client: &mut Client, _sender_id: ObjectId) -> Result<()> {
+		Ok(())
+	}
+
+	async fn set_parent_size(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_parent_width: i32,
+		_parent_height: i32,
+	) -> Result<()> {
+		let mut data = self.data.lock();
+		data.parent_size.x = _parent_width.max(0) as u32;
+		data.parent_size.y = _parent_height.max(0) as u32;
+		Ok(())
+	}
+
+	async fn set_parent_configure(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_serial: u32,
+	) -> Result<()> {
+		Ok(())
+	}
+
+	async fn destroy(&self, _client: &mut Client, _sender_id: ObjectId) -> Result<()> {
+		Ok(())
+	}
+}

src/wayland/xdg/surface.rs

@@ -0,0 +1,248 @@
+use super::{popup::Popup, positioner::Positioner, toplevel::Mapped};
+use crate::wayland::{core::surface::SurfaceRole, display::Display, xdg::toplevel::Toplevel};
+use waynest::server::protocol::stable::xdg_shell::xdg_popup::XdgPopup;
+use std::sync::Arc;
+use std::sync::Weak;
+pub use waynest::server::protocol::stable::xdg_shell::xdg_surface::*;
+use waynest::{
+	server::{Client, Dispatcher, Result},
+	wire::ObjectId,
+};
+
+#[derive(Debug, Dispatcher)]
+pub struct Surface {
+	id: ObjectId,
+	version: u32,
+	wl_surface: Weak<crate::wayland::core::surface::Surface>,
+	configured: Arc<std::sync::atomic::AtomicBool>,
+}
+impl Surface {
+	pub fn new(
+		id: ObjectId,
+		version: u32,
+		wl_surface: Arc<crate::wayland::core::surface::Surface>,
+	) -> Self {
+		Self {
+			id,
+			version,
+			wl_surface: Arc::downgrade(&wl_surface),
+			configured: Arc::new(std::sync::atomic::AtomicBool::new(false)),
+		}
+	}
+
+	pub fn wl_surface(&self) -> Arc<crate::wayland::core::surface::Surface> {
+		// We can safely unwrap as the surface must exist for the lifetime of the xdg_surface
+		self.wl_surface
+			.upgrade()
+			.expect("Surface was dropped before xdg_surface")
+	}
+
+	pub async fn reconfigure(&self, client: &mut Client) -> Result<()> {
+		let serial = client.next_event_serial();
+		self.configure(client, self.id, serial).await
+	}
+}
+
+impl XdgSurface for Surface {
+	/// https://wayland.app/protocols/xdg-shell#xdg_surface:request:destroy
+	async fn destroy(&self, _client: &mut Client, _sender_id: ObjectId) -> Result<()> {
+		Ok(())
+	}
+
+	/// https://wayland.app/protocols/xdg-shell#xdg_surface:request:get_toplevel
+	async fn get_toplevel(
+		&self,
+		client: &mut Client,
+		sender_id: ObjectId,
+		toplevel_id: ObjectId,
+	) -> Result<()> {
+		let surface = self.wl_surface();
+		let toplevel = client.insert(
+			toplevel_id,
+			Toplevel::new(
+				toplevel_id,
+				surface.clone(),
+				client.get::<Self>(sender_id).unwrap(),
+			),
+		);
+
+		{
+			let mut surface_role = surface.role.lock();
+
+			// A surface must not have any existing role when assigning a new one
+			// "A surface must not have more than one role, and a role must not be assigned to more than one
+			// surface at a time. However, wl_surface role-specific interfaces may reassign the role, allow
+			// a role to be destroyed, or allow multiple role-specific interfaces to share the same role."
+			// - xdg_surface protocol doc
+			if surface_role.is_some() {
+				// We should send "role" error here as per xdg_wm_base.error enum
+				// But we'll ignore for now
+			} else {
+				surface_role.replace(SurfaceRole::XdgToplevel(toplevel.clone()));
+			}
+		}
+
+		toplevel.reconfigure(client).await?;
+
+		let pid = client.get::<Display>(ObjectId::DISPLAY).unwrap().pid;
+		let configured = self.configured.clone();
+		surface.add_commit_handler(move |surface, state| {
+			let Some(SurfaceRole::XdgToplevel(toplevel)) = &mut *surface.role.lock() else {
+				return true;
+			};
+
+			// Only proceed if configured and has valid buffer
+			let has_valid_buffer = state
+				.buffer
+				.as_ref()
+				.is_some_and(|b| b.buffer.size().x > 0 && b.buffer.size().y > 0);
+
+			let mut mapped_lock = toplevel.mapped.lock();
+			if mapped_lock.is_none()
+				&& configured.load(std::sync::atomic::Ordering::SeqCst)
+				&& has_valid_buffer
+			{
+				mapped_lock.replace(Mapped::create(toplevel.clone(), pid));
+				return false;
+			}
+			true
+		});
+
+		Ok(())
+	}
+
+	/// https://wayland.app/protocols/xdg-shell#xdg_surface:request:get_popup
+	async fn get_popup(
+		&self,
+		client: &mut Client,
+		sender_id: ObjectId,
+		popup_id: ObjectId,
+		parent: Option<ObjectId>,
+		positioner: ObjectId,
+	) -> Result<()> {
+                let parent = client.get::<Surface>(parent.unwrap()).unwrap();
+                let panel_item = match parent.wl_surface().role.lock().as_ref().unwrap() {
+                        SurfaceRole::XdgToplevel(toplevel) => {
+                                let toplevel_lock = toplevel.mapped.lock();
+                                toplevel_lock.as_ref().unwrap()._panel_item.clone()
+                        }
+                        SurfaceRole::XDGPopup(popup) => popup.panel_item.upgrade().unwrap(),
+                };
+                let positioner = client.get::<Positioner>(positioner).unwrap();
+
+                let surface = client.get::<Surface>(self.id).unwrap();
+                let wl_surface = surface.wl_surface();
+
+                let popup = client.insert(
+                        popup_id,
+                        Popup::new(
+                                popup_id,
+                                self.version,
+                                parent,
+                                &panel_item,
+                                &surface,
+                                &positioner,
+                        ),
+                );
+
+                // Initial popup configure
+                let geometry = popup.current_geometry();
+                popup
+                        .configure(
+                                client,
+                                popup_id,
+                                geometry.origin.x,
+                                geometry.origin.y,
+                                geometry.size.x as i32,
+                                geometry.size.y as i32,
+                        )
+                        .await?;
+
+		{
+			let wl_surface = self.wl_surface();
+			let mut surface_role = wl_surface.role.lock();
+
+			if surface_role.is_some() {
+				// We should send "role" error here as per xdg_wm_base.error enum
+				// But we'll ignore for now
+			} else {
+				surface_role.replace(SurfaceRole::XDGPopup(popup.clone()));
+			}
+		}
+
+                let serial = client.next_event_serial();
+                self.configure(client, sender_id, serial).await?;
+
+                let configured = self.configured.clone();
+                wl_surface.add_commit_handler(move |surface, state| {
+                        let Some(SurfaceRole::XDGPopup(popup)) = &*surface.role.lock() else {
+                                return true;
+                        };
+                        let has_valid_buffer = state
+                                .buffer
+                                .as_ref()
+                                .is_some_and(|b| b.buffer.size().x > 0 && b.buffer.size().y > 0);
+                        if !popup.is_mapped()
+                                && configured.load(std::sync::atomic::Ordering::SeqCst)
+                                && has_valid_buffer
+                        {
+                                popup.map();
+                                return false;
+                        }
+                        true
+                });
+
+		// let pid = client.get::<Display>(ObjectId::DISPLAY).unwrap().pid;
+		// let configured = self.configured.clone();
+		// surface.add_commit_handler(move |surface, state| {
+		// 	let Some(SurfaceRole::XDGPopup(popup)) = &mut *surface.role.lock() else {
+		// 		return true;
+		// 	};
+
+		// 	// Only proceed if configured and has valid buffer
+		// 	let has_valid_buffer = state
+		// 		.buffer
+		// 		.as_ref()
+		// 		.is_some_and(|b| b.buffer.size().x > 0 && b.buffer.size().y > 0);
+
+		// 	let mut mapped_lock = popup.mapped.lock();
+		// 	if mapped_lock.is_none()
+		// 		&& configured.load(std::sync::atomic::Ordering::SeqCst)
+		// 		&& has_valid_buffer
+		// 	{
+		// 		mapped_lock.replace(Mapped::create(popup.clone(), pid));
+		// 		return false;
+		// 	}
+		// 	true
+		// });
+
+		Ok(())
+	}
+
+	/// https://wayland.app/protocols/xdg-shell#xdg_surface:request:set_window_geometry
+	async fn set_window_geometry(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_x: i32,
+		_y: i32,
+		_width: i32,
+		_height: i32,
+	) -> Result<()> {
+		// we're gonna delegate literally all the window management
+		// to 3D stuff sooo we don't care, maximized is the floating state
+		Ok(())
+	}
+
+	/// https://wayland.app/protocols/xdg-shell#xdg_surface:request:ack_configure
+	async fn ack_configure(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_serial: u32,
+	) -> Result<()> {
+		self.configured
+			.store(true, std::sync::atomic::Ordering::SeqCst);
+		Ok(())
+	}
+}

src/wayland/xdg/toplevel.rs

@@ -0,0 +1,310 @@
+use super::backend::XdgBackend;
+use crate::{
+	nodes::{Node, items::panel::PanelItem},
+	wayland::core::surface::Surface,
+};
+use mint::Vector2;
+use parking_lot::Mutex;
+use std::sync::Arc;
+use std::sync::Weak;
+pub use waynest::server::protocol::stable::xdg_shell::xdg_toplevel::*;
+use waynest::{
+	server::{Client, Dispatcher, Result},
+	wire::ObjectId,
+};
+
+#[derive(Debug)]
+pub struct Mapped {
+	pub panel_item_node: Arc<Node>,
+	pub _panel_item: Arc<PanelItem<XdgBackend>>,
+}
+impl Mapped {
+	pub fn create(toplevel: Arc<Toplevel>, pid: Option<i32>) -> Self {
+		let (panel_item_node, _panel_item) =
+			PanelItem::create(Box::new(XdgBackend::new(toplevel)), pid);
+
+		Self {
+			panel_item_node,
+			_panel_item,
+		}
+	}
+}
+
+#[derive(Debug, Clone)]
+struct ToplevelData {
+	parent: Option<u64>,
+	app_id: Option<String>,
+	title: Option<String>,
+	activated: bool,
+	fullscreen: bool,
+	pub size: Option<Vector2<u32>>,
+}
+impl Default for ToplevelData {
+	fn default() -> Self {
+		Self {
+			parent: None,
+			app_id: None,
+			title: None,
+			activated: true,
+			fullscreen: false,
+			size: None,
+		}
+	}
+}
+
+#[derive(Debug, Dispatcher)]
+pub struct Toplevel {
+	pub id: ObjectId,
+	wl_surface: Weak<Surface>,
+	xdg_surface: Weak<super::surface::Surface>,
+	pub mapped: Mutex<Option<Mapped>>,
+	data: Mutex<ToplevelData>,
+}
+impl Toplevel {
+	pub fn new(
+		object_id: ObjectId,
+		wl_surface: Arc<Surface>,
+		xdg_surface: Arc<super::surface::Surface>,
+	) -> Self {
+		Toplevel {
+			id: object_id,
+			wl_surface: Arc::downgrade(&wl_surface),
+			xdg_surface: Arc::downgrade(&xdg_surface),
+			mapped: Mutex::new(None),
+			data: Mutex::new(ToplevelData::default()),
+		}
+	}
+
+	pub fn surface(&self) -> Arc<Surface> {
+		// We can safely unwrap as the surface must exist for the lifetime of the toplevel
+		self.wl_surface
+			.upgrade()
+			.expect("Surface was dropped before toplevel")
+	}
+
+	pub fn title(&self) -> Option<String> {
+		self.data.lock().title.clone()
+	}
+	pub fn app_id(&self) -> Option<String> {
+		self.data.lock().app_id.clone()
+	}
+	pub fn parent(&self) -> Option<u64> {
+		self.data.lock().parent
+	}
+
+	pub fn set_size(&self, size: Option<Vector2<u32>>) {
+		self.data.lock().size = size;
+	}
+
+	pub fn set_activated(&self, activated: bool) {
+		self.data.lock().activated = activated;
+	}
+
+	// Helper to clamp size against constraints
+	fn clamp_size(&self, size: Vector2<u32>) -> Vector2<u32> {
+		let state = self.surface().current_state();
+		let mut clamped = size;
+
+		if let Some(min_size) = state.min_size {
+			clamped.x = clamped.x.max(min_size.x);
+			clamped.y = clamped.y.max(min_size.y);
+		}
+		if let Some(max_size) = state.max_size {
+			clamped.x = clamped.x.min(max_size.x);
+			clamped.y = clamped.y.min(max_size.y);
+		}
+		clamped
+	}
+
+	pub async fn reconfigure(&self, client: &mut Client) -> Result<()> {
+		let data = self.data.lock().clone();
+
+		// Use the explicitly set size, applying constraints
+		let size = data.size.map(|s| self.clamp_size(s));
+
+		let mut states = vec![
+			State::TiledTop,
+			State::TiledLeft,
+			State::TiledRight,
+			State::TiledBottom,
+			if data.fullscreen {
+				State::Fullscreen
+			} else {
+				State::Maximized
+			},
+		];
+		if data.activated {
+			states.push(State::Activated);
+		}
+
+		self.configure(
+			client,
+			self.id,
+			size.map(|v| v.x as i32).unwrap_or(0),
+			size.map(|v| v.y as i32).unwrap_or(0),
+			states
+				.into_iter()
+				.flat_map(|x| (x as u32).to_ne_bytes())
+				.collect(),
+		)
+		.await?;
+		self.xdg_surface
+			.upgrade()
+			.unwrap()
+			.reconfigure(client)
+			.await
+	}
+}
+impl XdgToplevel for Toplevel {
+	async fn set_parent(
+		&self,
+		client: &mut Client,
+		_sender_id: ObjectId,
+		parent: Option<ObjectId>,
+	) -> Result<()> {
+		// Handle case where parent is specified
+		if let Some(parent) = parent {
+			// Per spec: parent must be another xdg_toplevel surface
+			if let Some(parent_toplevel) = client.get::<Toplevel>(parent) {
+				let Some(mapped) = &*parent_toplevel.mapped.lock() else {
+					// Per spec: parent surfaces must be mapped before being used as a parent
+					// Setting an unmapped window as parent should raise a protocol error
+					// For now we just unset the parent as a fallback
+					self.data.lock().parent.take();
+					return Ok(());
+				};
+
+				// Per spec: store parent to ensure this surface is stacked above parent
+				// and other ancestor surfaces. Used for proper window stacking order.
+				self.data
+					.lock()
+					.parent
+					.replace(mapped.panel_item_node.get_id());
+			}
+		} else {
+			// Per spec: null parent unsets the parent, making this a top-level window
+			// This allows converting child windows back to independent top-level windows
+			self.data.lock().parent.take();
+		}
+
+		Ok(())
+	}
+
+	async fn set_title(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		title: String,
+	) -> Result<()> {
+		self.data.lock().title.replace(title);
+		Ok(())
+	}
+
+	async fn set_app_id(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		app_id: String,
+	) -> Result<()> {
+		self.data.lock().app_id.replace(app_id);
+		Ok(())
+	}
+
+	async fn show_window_menu(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_seat: ObjectId,
+		_serial: u32,
+		_x: i32,
+		_y: i32,
+	) -> Result<()> {
+		Ok(())
+	}
+
+	async fn r#move(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_seat: ObjectId,
+		_serial: u32,
+	) -> Result<()> {
+		Ok(())
+	}
+
+	async fn resize(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_seat: ObjectId,
+		_serial: u32,
+		_edges: ResizeEdge,
+	) -> Result<()> {
+		Ok(())
+	}
+
+	async fn set_max_size(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		width: i32,
+		height: i32,
+	) -> Result<()> {
+		self.surface().pending_state().pending.max_size = if width == 0 && height == 0 {
+			None
+		} else {
+			Some([width as u32, height as u32].into())
+		};
+		Ok(())
+	}
+
+	async fn set_min_size(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		width: i32,
+		height: i32,
+	) -> Result<()> {
+		self.surface().pending_state().pending.min_size = if width == 0 && height == 0 {
+			None
+		} else {
+			Some([width as u32, height as u32].into())
+		};
+		Ok(())
+	}
+
+	async fn set_maximized(&self, _client: &mut Client, _sender_id: ObjectId) -> Result<()> {
+		Ok(())
+	}
+
+	async fn unset_maximized(&self, _client: &mut Client, _sender_id: ObjectId) -> Result<()> {
+		Ok(())
+	}
+
+	async fn set_fullscreen(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_output: Option<ObjectId>,
+	) -> Result<()> {
+		Ok(())
+	}
+
+	async fn unset_fullscreen(&self, _client: &mut Client, _sender_id: ObjectId) -> Result<()> {
+		Ok(())
+	}
+
+	async fn set_minimized(&self, _client: &mut Client, _sender_id: ObjectId) -> Result<()> {
+		Ok(())
+	}
+
+	async fn destroy(&self, _client: &mut Client, _sender_id: ObjectId) -> Result<()> {
+		self.mapped.lock().take();
+		Ok(())
+	}
+}
+impl Drop for Toplevel {
+	fn drop(&mut self) {
+		self.mapped.lock().take();
+	}
+}

src/wayland/xdg/wm_base.rs

@@ -0,0 +1,50 @@
+use crate::wayland::xdg::surface::Surface;
+pub use waynest::server::protocol::stable::xdg_shell::xdg_wm_base::*;
+use waynest::{
+	server::{Client, Dispatcher, Result},
+	wire::ObjectId,
+};
+
+use super::positioner::Positioner;
+
+#[derive(Debug, Dispatcher, Default)]
+pub struct WmBase {
+	pub version: u32,
+}
+impl XdgWmBase for WmBase {
+	async fn destroy(&self, _client: &mut Client, _sender_id: ObjectId) -> Result<()> {
+		Ok(())
+	}
+
+	async fn create_positioner(
+		&self,
+		client: &mut Client,
+		_sender_id: ObjectId,
+		id: ObjectId,
+	) -> Result<()> {
+		client.insert(id, Positioner::default());
+		Ok(())
+	}
+
+	async fn get_xdg_surface(
+		&self,
+		client: &mut Client,
+		_sender_id: ObjectId,
+		xdg_surface_id: ObjectId,
+		wl_surface_id: ObjectId,
+	) -> Result<()> {
+		let wl_surface = client
+			.get::<crate::wayland::core::surface::Surface>(wl_surface_id)
+			.ok_or(waynest::server::Error::Custom(
+				"can't get wayland surface id".to_string(),
+			))?;
+		let xdg_surface = Surface::new(xdg_surface_id, self.version, wl_surface);
+		client.insert(xdg_surface_id, xdg_surface);
+
+		Ok(())
+	}
+
+	async fn pong(&self, _client: &mut Client, _sender_id: ObjectId, _serial: u32) -> Result<()> {
+		Ok(())
+	}
+}