Code style

Signed-off-by: Schmarni <marnistromer@gmail.com>

.cargo/config.toml

@@ -0,0 +1,2 @@
+[build]
+rustflags = ["--cfg", "tokio_unstable"]

.github/FUNDING.yml

@@ -0,0 +1,13 @@
+# These are supported funding model platforms
+
+github: [technobaboo]
+patreon: # Replace with a single Patreon username
+open_collective: # Replace with a single Open Collective username
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry
+custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

.github/workflows/build.yml

@@ -0,0 +1,26 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - "*"
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install runtime dependencies
+        run: sudo apt install -y --no-install-recommends libopenxr-dev libwayland-dev libasound2-dev
+
+      - name: Set up Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: 1.88.0
+          override: true
+
+      - name: Build server
+        run: cargo build --release

.gitignore

@@ -2,11 +2,15 @@
 # will have compiled files and executables
 /target/
 
-# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
-# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html
-Cargo.lock
-
 # These are backup files generated by rustfmt
 **/*.rs.bk
 
 .vscode/
+
+# Ignore build results from Nix
+*result*
+
+/libs/
+*.AppImage
+*.blend1
+anchors.txt

.lapce/run.toml

@@ -0,0 +1,29 @@
+# The run config is used for both run mode and debug mode
+
+[[configs]]
+# the name of this task
+name = "task"
+
+# the type of the debugger. If not set, it can't be debugged but can still be run
+# type = "lldb"
+
+# the program to run
+program = "./target/debug/stardust-xr-server"
+
+# the program arguments, e.g. args = ["arg1", "arg2"], optional
+# args = []
+
+# current working directory, optional
+cwd = "${workspace}"
+
+# enviroment variables, optional
+# [configs.env]
+# VAR1 = "VAL1"
+# VAR2 = "VAL2"
+
+# task to run before the run/debug session is started, optional
+# [configs.prelaunch]
+# program = "cargo"
+# args = [
+#   "build",
+# ]

.zed/debug.json

@@ -0,0 +1,94 @@
+[
+	{
+		"label": "Debug",
+		"adapter": "CodeLLDB",
+		"program": "target/debug/stardust-xr-server",
+		"args": [],
+		"env": {
+			"RUST_LOG": "debug"
+		},
+		"request": "launch",
+		"build": {
+			"command": "cargo",
+			"args": [
+				"build",
+				"--bin=stardust-xr-server",
+				"--package=stardust-xr-server"
+			]
+		}
+	},
+	{
+		"label": "Debug (Overlay)",
+		"adapter": "CodeLLDB",
+		"program": "target/debug/stardust-xr-server",
+		"args": ["-o", "1"],
+		"env": {
+			"RUST_LOG": "debug"
+		},
+		"request": "launch",
+		"build": {
+			"command": "cargo",
+			"args": [
+				"build",
+				"--bin=stardust-xr-server",
+				"--package=stardust-xr-server"
+			]
+		}
+	},
+	{
+		"label": "Debug (Headless)",
+		"adapter": "CodeLLDB",
+		"program": "target/debug/stardust-xr-server",
+		"args": [],
+		"env": {
+			"RUST_LOG": "debug",
+			"DISPLAY": "",
+			"WAYLAND_DISPLAY": ""
+		},
+		"request": "launch",
+		"build": {
+			"command": "cargo",
+			"args": [
+				"build",
+				"--bin=stardust-xr-server",
+				"--package=stardust-xr-server"
+			]
+		}
+	},
+	{
+		"label": "Debug (Flatscreen)",
+		"adapter": "CodeLLDB",
+		"program": "target/debug/stardust-xr-server",
+		"args": ["-f"],
+		"env": {
+			"RUST_LOG": "debug"
+		},
+		"request": "launch",
+		"build": {
+			"command": "cargo",
+			"args": [
+				"build",
+				"--bin=stardust-xr-server",
+				"--package=stardust-xr-server"
+			]
+		}
+	},
+	{
+		"label": "Debug (Flatscreen, Restore Latest)",
+		"adapter": "CodeLLDB",
+		"program": "target/debug/stardust-xr-server",
+		"args": ["-f", "--restore", "latest"],
+		"env": {
+			"RUST_LOG": "debug"
+		},
+		"request": "launch",
+		"build": {
+			"command": "cargo",
+			"args": [
+				"build",
+				"--bin=stardust-xr-server",
+				"--package=stardust-xr-server"
+			]
+		}
+	}
+]

Cargo.toml

@@ -1,51 +1,165 @@
 [package]
-edition = "2018"
+edition = "2024"
+rust-version = "1.88"
 name = "stardust-xr-server"
-version = "0.10.1"
+version = "0.45.0"
 authors = ["Nova King <technobaboo@proton.me>"]
 description = "Stardust XR reference display server"
 license = "GPLv2"
 repository = "https://github.com/StardustXR/stardust-xr-server/"
 homepage = "https://stardustxr.org"
 
+[workspace]
+members = ["codegen"]
+
+[workspace.dependencies.stardust-xr]
+git = "https://github.com/StardustXR/core.git"
+branch = "dev"
+
+[[bin]]
+name = "stardust-xr-server"
+path = "src/main.rs"
+
+[features]
+default = ["wayland"]
+wayland = [
+	"dep:cluFlock",
+	"dep:waynest",
+	"dep:tokio-stream",
+	"dep:memmap2",
+	"dep:drm-fourcc",
+	"dep:memfd",
+	"dep:vulkano",
+	"dep:wgpu-hal",
+	"dep:ash",
+]
+profile_tokio = ["dep:console-subscriber", "tokio/tracing"]
+profile_app = ["dep:tracing-tracy", "bevy/trace_tracy", "bevy/trace"]
+bevy_debugging = ["bevy/bevy_remote", "bevy/track_location"]
+
+[package.metadata.appimage]
+auto_link = true
+auto_link_exclude_list = ["libc*", "libdl*", "libpthread*", "ld-linux*"]
+
+
+[profile.release]
+lto = true
+strip = true
+opt-level = 3
+
+[profile.dev.package."*"]
+opt-level = 3
+
+[patch.crates-io]
+bevy_mod_openxr = { git = "https://github.com/awtterpip/bevy_oxr" }
+bevy_mod_xr = { git = "https://github.com/awtterpip/bevy_oxr" }
+bevy_gltf = { git = "https://github.com/Schmarni-Dev/bevy", branch = "gltf_backport" }
+# TODO: figure out how to not need this horrifing patch
+wgpu = { git = "https://github.com/Schmarni-Dev/wgpu", branch = "bad_dmabuf_workaround" }
+wgpu-types = { git = "https://github.com/Schmarni-Dev/wgpu", branch = "bad_dmabuf_workaround" }
+wgpu-core = { git = "https://github.com/Schmarni-Dev/wgpu", branch = "bad_dmabuf_workaround" }
+naga = { git = "https://github.com/Schmarni-Dev/wgpu", branch = "bad_dmabuf_workaround" }
+wgpu-hal = { git = "https://github.com/Schmarni-Dev/wgpu", branch = "bad_dmabuf_workaround" }
+
 [dependencies]
-anyhow = "1.0.57"
-clap = { version = "4.0.8", features = ["derive"] }
-ctrlc = "3.2.2"
-dashmap = "5.3.4"
-flatbuffers = "2.1.2"
-flexbuffers = "2.0.0"
-glam = {version = "0.21.3", features = ["mint"]}
-lazy_static = "1.4.0"
-mint = "0.5.9"
+# small utility thingys
 nanoid = "0.4.0"
-once_cell = "1.12.0"
-parking_lot = "0.12.1"
-portable-atomic = {version = "0.3.0", features = ["float", "std"]}
-rccell = "0.1.3"
-rustc-hash = "1.1.0"
-slab = "0.4.6"
-tokio = { version = "1", features = ["full"] }
-thiserror = "1.0.31"
-send_wrapper = "0.6.0"
-prisma = "0.1.1"
-slog = "2.7.0"
-slog-stdlog = "4.1.1"
-xkbcommon = { version = "0.5.0", default-features = false }
-stardust-xr = "0.5.2"
-wayland-backend = "=0.1.0-beta.9"
-wayland-scanner = "=0.30.0-beta.9"
-directories = "4.0.1"
-serde = { version = "1.0.145", features = ["derive"] }
+lazy_static = "1.5.0"
+rand = "0.8.5"
+rustc-hash = "2.0.0"
+slotmap = "1.0.7"
+global_counter = "=0.2.2"
+parking_lot = "0.12.3"
+dashmap = "6.1.0"
 
-[dependencies.stereokit]
-default-features = false
-features = ["linux-egl"]
-version = "0.5.0"
+# rust errors/logging
+color-eyre = { version = "0.6.3", default-features = false }
+clap = { version = "4.5.13", features = ["derive"] }
+console-subscriber = { version = "0.4.0", optional = true }
+thiserror = "2.0.11"
+tracing = { version = "0.1.40", features = ["release_max_level_warn"] }
+tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
+tracing-tracy = { version = "0.11.1", optional = true }
+tracy-client = { version = "=0.18.0" }
 
-[dependencies.smithay]
-git = "https://github.com/technobaboo/smithay.git"
-branch = "feature/public_input"
-default-features = false
-features = ["desktop", "renderer_gl", "wayland_frontend"]
-version = "*"
+# (de)serialization
+serde = { version = "1.0.205", features = ["derive"] }
+serde_repr = "0.1.19"
+toml = "0.8.19"
+
+# mathy stuffs
+glam = { version = "0.29.0", features = ["mint", "serde"] }
+mint = "0.5.9"
+tokio = { version = "1.39.2", features = ["rt-multi-thread", "signal", "time"] }
+
+# bevy
+bevy = { version = "0.16", default-features = false, features = [
+	"bevy_asset",
+	"bevy_audio",
+	"bevy_color",
+	"bevy_core_pipeline",
+	"bevy_gizmos",
+	"bevy_gltf",
+	"bevy_log",
+	"bevy_pbr",
+	"bevy_render",
+	"bevy_window",
+	"bevy_winit",
+	"std",
+	"x11",
+	"wayland",
+	"mp3",
+	"wav",
+	"qoi",
+	"png",
+	"hdr",
+	"jpeg",
+	"tonemapping_luts",
+	"multi_threaded",
+] }
+bevy_mod_xr = "0.3"
+bevy_mod_openxr = "0.3"
+# bevy_sk.git = "https://github.com/MalekiRe/bevy_sk"
+bevy_sk = { git = "https://github.com/technobaboo/bevy_sk", branch = "stochastic" }
+# bevy-dmabuf = "0.2.0"
+bevy-dmabuf.git = "https://github.com/Schmarni-Dev/bevy-dmabuf"
+
+# bevy-mesh-text-3d.git = "https://github.com/terhechte/bevy-mesh-text-3d"
+# use my fork until my pr to use minimal bevy features was merged
+bevy-mesh-text-3d.git = "https://github.com/Schmarni-Dev/bevy-mesh-text-3d"
+
+openxr = "0.19"
+
+# linux stuffs
+input-event-codes = "6.2.0"
+zbus = { version = "5.0.0", default-features = false, features = ["tokio"] }
+directories = "5.0.1"
+xkbcommon-rs = "0.1.0"
+cosmic-text = "0.14.2"
+
+# Wayland
+cluFlock = { version = "1.2.7", optional = true } # for the lockfile checking
+# waynest = { git = "https://github.com/verdiwm/waynest.git", features = [
+# 	"server",
+# 	"stable",
+# 	"tracing",
+# ], default-features = false, optional = true }
+waynest = { git = "https://github.com/technobaboo/waynest.git", branch = "fix/wayland-drm", features = [
+	"server",
+	"stable",
+	"external",
+	"tracing",
+], default-features = false, optional = true }
+tokio-stream = { version = "0.1.17", optional = true }
+memmap2 = { version = "0.9.5", optional = true }
+drm-fourcc = { version = "2.2.0", optional = true }
+memfd = { version = "0.6.4", optional = true }
+vulkano = { git = "https://github.com/Schmarni-Dev/vulkano", branch = "0_35_dmabuf_fixes", optional = true }
+wgpu-hal = { version = "24", optional = true, features = ["vulkan"] }
+ash = { version = "0.38.0", optional = true, default-features = false }
+rustix = { version = "1.0.8", features = ["time"] }
+
+[dependencies.stardust-xr]
+workspace = true
+[dependencies.stardust-xr-server-codegen]
+path = "codegen"

README.md

@@ -1,22 +1,113 @@
-# Stardust XR Reference Server
+# Stardust XR Server
 
-This project is a usable Linux display server that reinvents human-computer interaction for all kinds of XR, from putting 2D/XR apps into various 3D shells for varying uses to SDF-based interaction.
+Stardust XR is a display server for VR and AR headsets on Linux-based systems. [Stardust provides a 3D environment](https://www.youtube.com/watch?v=v2WblwbaLaA), where anything from 2D windows (including your existing apps!), to 3D apps built from objects, can exist together in physical space.
 
-## Prerequisites
-1. Cargo
-2. CMake
-3. EGL+GLES 3.2
-4. GLX+Xlib
-5. fontconfig
-6. dlopen
-7. OpenXR Loader (required even if run in flatscreen mode)
+![workflow](/img/workflow.png)
+
+Command line installation of core & dynamic dependencies are provided below:
+<details>
+<summary>Ubuntu/Debian</summary>
+  <pre><code class="language-bash">
+  sudo apt update && sudo apt install \
+  build-essential \
+  cargo \
+  cmake \
+  libxkbcommon-dev libudev1 libinput10 libcap2 libmtdev1 libevdev2 libwacom9 libgudev-1.0-0 \
+  libglib2.0-dev libffi8 libpcre2-dev libxkbcommon-x11-dev libxcb-dev libxcb-xkb-dev libxau-dev \ 
+  libstdc++-dev libx11-dev libxfixes-dev libegl-dev libgbm-dev libfontconfig1-dev libgl-dev \
+  libdrm-dev libexpat1-dev libfreetype6-dev libxml2-dev zlib1g-dev libbz2-dev libpng-dev \ 
+  libharfbuzz-dev libbrotli-dev liblzma-dev libraphite2-dev
+  </code></pre>
+</details>
+
+<details>
+<summary>Fedora</summary>
+  <pre><code class="language-bash">
+  sudo apt update && sudo apt install \
+  cargo \
+  cmake \
+  libxkbcommon-devel systemd-devel libinput-devel libcap-devel mtdev-devel libevdev-devel glib2-devel \
+  libffi-devel pcre2-devel libxkbcommon-x11-devel libxcb-devel libXau-devel libstdc++-devel libx11-devel libxfixes-devel \
+  mesa-libEGL-devel mesa-libgbm-devel fontconfig-devel libdrm-devel expat-devel freetype-devel libxml2-devel zlib-devel \
+  bzip2-devel libpng-devel harfbuzz-devel brotli-devel xz-devel graphite2-devel
+  </code></pre>
+</details>
+
+
+<details>
+<summary>Arch Linux</summary>
+  <pre><code class="language-bash">
+  sudo pacman -Syu --needed \
+  cargo \
+  cmake \
+  libxkbcommon systemd libinput libcap mtdev libevdev libwacom glib2 libffi pcre2 libxkbcommon-x11 \
+  libxcb libxau libx11 libxfixes mesa fontconfig libdrm expat freetype2 libxml2 zlib bzip2 \ 
+  libpng harfbuzz brotli xz graphite
+  </code></pre>
+</details>
+
+## Installation
+
+More detailed instructions and walkthroughs are provided at https://www.stardustxr.org
+
+The [Terra Repository](https://terra.fyralabs.com/) is required, and comes pre-installed with [Ultramarine Linux](https://ultramarine-linux.org/). Other Fedora Editions and derivatives can directly install terra-release:
 
-## Build
 ```bash
-cargo build
+sudo dnf install --nogpgcheck --repofrompath 'terra,https://repos.fyralabs.com/terra$releasever' terra-release
 ```
 
-## Install
+For a full installation of the Stardust XR server *and* a selected group of clients, run:
+
 ```bash
-cargo install
-```
+sudo dnf group install stardust-xr
+```
+
+## Manual Build
+We've provided a manual installation script [here](https://github.com/cyberneticmelon/usefulscripts/blob/main/stardustxr_setup.sh) that clones and builds the Stardust XR server along with a number of other clients from their respective repositories, and provides a startup script for automatically launching some clients.
+
+After cloning the repository
+```bash
+cargo build --release # this is needed to skip validation layers
+```
+
+## Usage
+> [!NOTE]
+> For help with setting up an XR headset on linux, visit https://stardustxr.org/docs/get-started/setup-openxr
+
+
+The **Stardust XR Server** is a server that runs clients, so without any running, you will see a black screen. If you only have the server installed, we recommend also cloning and building the following clients to start: [Flatland](https://github.com/StardustXR/flatland), which allows normal 2D apps to run in Stardust, [Protostar](https://github.com/StardustXR/protostar), which contains Hexagon Launcher, an app launcher menu, and [Black Hole](https://github.com/StardustXR/black-hole) to quickly tuck away your objects and apps (kind of like desktop peek on Windows).
+
+First, try running `cargo run -- -f` in a terminal window to check out flatscreen mode, (or `stardust-xr-server -f` / `stardust-xr-server_dev -f` if you installed via dnf or the manual installation script, respectively, as they provide symlinks.)
+
+If there aren't already any clients running, you'll need to manually launch them by either navigating to their repositories and running `cargo run`, or running them via their names if you installed via dnf or the manual installation script, such as `flatland`, `hexagon_launcher`, etc.
+
+> [!IMPORTANT]
+> [Flatland](https://github.com/StardustXR/flatland) must be running for 2D apps to launch.
+
+### Startup Script
+A startup script can be created at `~/.config/stardust/startup` that will launch specified settings and clients/applications, an example of which is shown [here](https://github.com/cyberneticmelon/usefulscripts/blob/main/startup). If you used the [installation script](https://github.com/cyberneticmelon/usefulscripts/blob/main/stardustxr_setup.sh), one will have already been made for you. This allows wide flexibility of what clients to launch upon startup (and, for example, *where*, using the [Gravity](https://github.com/StardustXR/gravity) client to specify X Y and Z co-ordinates).
+
+### Flatscreen Navigation
+A video guide showcasing flatscreen controls is available [here](https://www.youtube.com/watch?v=JCYecSlKlDI)
+
+To move around, hold down `Shift + W A S D`, with `Q` for moving down and `E` for moving up.
+![wasd](https://github.com/StardustXR/website/blob/main/static/img/updated_flat_wasd.GIF)
+
+To look around, hold down `Shift + Right` Click while moving the mouse.
+![updated_look](https://github.com/StardustXR/website/blob/main/static/img/updated_flat_look.GIF)
+
+To drag applications out of the app launcher, hold down `Shift + ~`
+![updated_drag](https://github.com/StardustXR/website/blob/main/static/img/updated_flat_drag.GIF)
+
+### XR Navigation
+A video guide showcasing XR controls is available [here](https://www.youtube.com/watch?v=RbxFq6JjliA)
+
+**Quest 3 Hand tracking**:
+Pinch to drag and drop, grasp with full hand for grabbing, point and click with pointer finger to click or pinch from a distance
+
+![hand_pinching](https://github.com/StardustXR/website/blob/main/static/img/hand_pinching.GIF)
+
+**Quest 3 Controller**:
+Grab with the grip buttons, click by touching the tip of the cones or by using the trigger from a distance
+
+![controller_click](https://github.com/StardustXR/website/blob/main/static/img/controller_click.GIF)

codegen/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+edition = "2021"
+name = "stardust-xr-server-codegen"
+version = "0.1.0"
+
+[lib]
+proc-macro = true
+
+[dependencies]
+convert_case = "0.6.0"
+quote = "1.0.33"
+mint = "0.5.9"
+proc-macro2 = "1.0.71"
+split-iter = "0.1.0"
+
+stardust-xr = { workspace = true }

codegen/src/lib.rs

@@ -0,0 +1,724 @@
+use convert_case::{Case, Casing};
+use proc_macro2::{Ident, Span, TokenStream};
+use quote::{quote, ToTokens};
+use split_iter::Splittable;
+use stardust_xr::schemas::protocol::*;
+
+fn fold_tokens(a: TokenStream, b: TokenStream) -> TokenStream {
+	quote!(#a #b)
+}
+
+#[proc_macro]
+pub fn codegen_root_protocol(_input: proc_macro::TokenStream) -> proc_macro::TokenStream {
+	codegen_protocol(ROOT_PROTOCOL)
+}
+#[proc_macro]
+pub fn codegen_node_protocol(_input: proc_macro::TokenStream) -> proc_macro::TokenStream {
+	codegen_protocol(NODE_PROTOCOL)
+}
+#[proc_macro]
+pub fn codegen_spatial_protocol(_input: proc_macro::TokenStream) -> proc_macro::TokenStream {
+	codegen_protocol(SPATIAL_PROTOCOL)
+}
+#[proc_macro]
+pub fn codegen_field_protocol(_input: proc_macro::TokenStream) -> proc_macro::TokenStream {
+	codegen_protocol(FIELD_PROTOCOL)
+}
+#[proc_macro]
+pub fn codegen_audio_protocol(_input: proc_macro::TokenStream) -> proc_macro::TokenStream {
+	codegen_protocol(AUDIO_PROTOCOL)
+}
+#[proc_macro]
+pub fn codegen_drawable_protocol(_input: proc_macro::TokenStream) -> proc_macro::TokenStream {
+	codegen_protocol(DRAWABLE_PROTOCOL)
+}
+#[proc_macro]
+pub fn codegen_input_protocol(_input: proc_macro::TokenStream) -> proc_macro::TokenStream {
+	codegen_protocol(INPUT_PROTOCOL)
+}
+#[proc_macro]
+pub fn codegen_item_protocol(_input: proc_macro::TokenStream) -> proc_macro::TokenStream {
+	codegen_protocol(ITEM_PROTOCOL)
+}
+#[proc_macro]
+pub fn codegen_item_camera_protocol(_input: proc_macro::TokenStream) -> proc_macro::TokenStream {
+	codegen_protocol(ITEM_CAMERA_PROTOCOL)
+}
+#[proc_macro]
+pub fn codegen_item_panel_protocol(_input: proc_macro::TokenStream) -> proc_macro::TokenStream {
+	codegen_protocol(ITEM_PANEL_PROTOCOL)
+}
+
+fn codegen_protocol(protocol: &'static str) -> proc_macro::TokenStream {
+	let protocol = Protocol::parse(protocol).unwrap();
+	let interface = protocol
+		.interface
+		.map(|p| {
+			let node_id = p.node_id;
+			let node_id = quote! {
+				const INTERFACE_NODE_ID: u64 = #node_id;
+			};
+			let aspect = generate_aspect(&Aspect {
+				name: "interface".to_string(),
+				id: 0,
+				description: protocol.description.clone(),
+				inherits: vec![],
+				members: p.members,
+			});
+			quote! {
+				#node_id
+				#aspect
+				pub struct Interface;
+				impl crate::nodes::AspectIdentifier for Interface {
+					impl_aspect_for_interface_aspect_id!{}
+				}
+				impl crate::nodes::Aspect for Interface {
+					impl_aspect_for_interface_aspect!{}
+				}
+				pub fn create_interface(client: &std::sync::Arc<crate::core::client::Client>) -> crate::core::error::Result<()>{
+					let node = crate::nodes::Node::from_id(client,INTERFACE_NODE_ID,false);
+					node.add_aspect(Interface);
+					node.add_to_scenegraph()?;
+					Ok(())
+				}
+			}
+		})
+		.unwrap_or_default();
+	let custom_enums = protocol
+		.custom_enums
+		.iter()
+		.map(generate_custom_enum)
+		.reduce(fold_tokens)
+		.unwrap_or_default();
+	let custom_unions = protocol
+		.custom_unions
+		.iter()
+		.map(generate_custom_union)
+		.reduce(fold_tokens)
+		.unwrap_or_default();
+	let custom_structs = protocol
+		.custom_structs
+		.iter()
+		.map(generate_custom_struct)
+		.reduce(fold_tokens)
+		.unwrap_or_default();
+	let aspects = protocol
+		.aspects
+		.iter()
+		.map(generate_aspect)
+		.reduce(fold_tokens)
+		.unwrap_or_default();
+	quote!(#custom_enums #custom_unions #custom_structs #aspects #interface).into()
+}
+
+fn generate_custom_enum(custom_enum: &CustomEnum) -> TokenStream {
+	let name = Ident::new(&custom_enum.name.to_case(Case::Pascal), Span::call_site());
+	let description = &custom_enum.description;
+
+	let argument_decls = custom_enum
+		.variants
+		.iter()
+		.map(|a| Ident::new(&a.to_case(Case::Pascal), Span::call_site()).to_token_stream())
+		.reduce(|a, b| quote!(#a, #b))
+		.unwrap_or_default();
+
+	quote! {
+		#[doc = #description]
+		#[derive(Debug, Clone, Copy, serde_repr::Deserialize_repr, serde_repr::Serialize_repr)]
+		#[repr(u32)]
+		pub enum #name {#argument_decls}
+	}
+}
+fn generate_custom_union(custom_union: &CustomUnion) -> TokenStream {
+	let name = Ident::new(&custom_union.name.to_case(Case::Pascal), Span::call_site());
+	let description = &custom_union.description;
+
+	let option_decls = custom_union
+		.options
+		.iter()
+		.map(generate_union_option)
+		.reduce(|a, b| quote!(#a, #b))
+		.unwrap_or_default();
+
+	quote! {
+		#[doc = #description]
+		#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)]
+		#[serde(tag = "t", content = "c")]
+		pub enum #name {#option_decls}
+	}
+}
+fn generate_union_option(union_option: &UnionOption) -> TokenStream {
+	let name = union_option
+		.name
+		.as_ref()
+		.map(|n| n.to_case(Case::Pascal))
+		.unwrap_or_else(|| argument_type_option_name(&union_option._type));
+	let description = union_option
+		.description
+		.as_ref()
+		.map(|d| quote!(#[doc = #d]))
+		.unwrap_or_default();
+	let identifier = Ident::new(&name, Span::call_site());
+	let _type = generate_argument_type(&union_option._type, false, true);
+	quote! (#description #identifier(#_type))
+}
+fn generate_custom_struct(custom_struct: &CustomStruct) -> TokenStream {
+	let name = Ident::new(&custom_struct.name.to_case(Case::Pascal), Span::call_site());
+	let description = &custom_struct.description;
+
+	let argument_decls = custom_struct
+		.fields
+		.iter()
+		.map(|a| generate_argument_decl(a, true))
+		.map(|d| quote!(pub #d))
+		.reduce(|a, b| quote!(#a, #b))
+		.unwrap_or_default();
+
+	quote! {
+		#[doc = #description]
+		#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)]
+		pub struct #name {#argument_decls}
+	}
+}
+
+fn generate_aspect(aspect: &Aspect) -> TokenStream {
+	let description = &aspect.description;
+	let (client_members, server_members) = aspect.members.iter().split(|m| m.side == Side::Server);
+
+	let client_mod_name = Ident::new(
+		&format!("{}_client", &aspect.name.to_case(Case::Snake)),
+		Span::call_site(),
+	);
+	let client_side_members = client_members
+		.map(|m| generate_member(aspect.id, &aspect.name.to_case(Case::Snake), m))
+		.reduce(fold_tokens)
+		.map(|t| {
+			// TODO: properly import all dependencies
+			quote! {
+				#[allow(clippy::all)]
+				pub mod #client_mod_name {
+					use super::*;
+					#t
+				}
+			}
+		})
+		.unwrap_or_default();
+
+	let opcodes = aspect
+		.members
+		.iter()
+		.map(|m| {
+			let aspect_name = aspect.name.to_case(Case::ScreamingSnake);
+			let member_name = m.name.to_case(Case::ScreamingSnake);
+			let name_type = if m.side == Side::Client {
+				"CLIENT"
+			} else {
+				"SERVER"
+			};
+			let name = Ident::new(
+				&format!("{aspect_name}_{member_name}_{name_type}_OPCODE"),
+				Span::call_site(),
+			);
+			let opcode = m.opcode;
+
+			quote!(pub(crate) const #name: u64 = #opcode;)
+		})
+		.reduce(fold_tokens)
+		.unwrap_or_default();
+	let alias_info = generate_alias_info(aspect);
+
+	let server_side_members = server_members
+		.map(|m| generate_member(aspect.id, &aspect.name.to_case(Case::Pascal), m))
+		.reduce(fold_tokens)
+		.unwrap_or_default();
+	let aspect_trait_name = Ident::new(
+		&format!("{}Aspect", &aspect.name.to_case(Case::Pascal)),
+		Span::call_site(),
+	);
+	let run_signals = aspect
+		.members
+		.iter()
+		.filter(|m| m.side == Side::Server)
+		.filter(|m| m._type == MemberType::Signal)
+		.map(|m| generate_run_member(&aspect_trait_name, MemberType::Signal, m))
+		.reduce(fold_tokens)
+		.unwrap_or_default();
+	let run_methods = aspect
+		.members
+		.iter()
+		.filter(|m| m.side == Side::Server)
+		.filter(|m| m._type == MemberType::Method)
+		.map(|m| generate_run_member(&aspect_trait_name, MemberType::Method, m))
+		.reduce(fold_tokens)
+		.unwrap_or_default();
+	let server_side_members = quote! {
+		#[allow(clippy::all)]
+		#[doc = #description]
+		pub trait #aspect_trait_name {
+			#server_side_members
+		}
+	};
+	let aspect_id_macro_name = Ident::new(
+		&format!(
+			"impl_aspect_for_{}_aspect_id",
+			aspect.name.to_case(Case::Snake)
+		),
+		Span::call_site(),
+	);
+	let aspect_macro_name = Ident::new(
+		&format!(
+			"impl_aspect_for_{}_aspect",
+			aspect.name.to_case(Case::Snake)
+		),
+		Span::call_site(),
+	);
+	let aspect_id = aspect.id;
+	let aspect_macro = quote! {
+		macro_rules! #aspect_id_macro_name {
+			() => {
+				const ID: u64 = #aspect_id;
+			}
+		}
+		macro_rules! #aspect_macro_name {
+			() => {
+				fn as_any(self: Arc<Self>) -> Arc<dyn std::any::Any + Send + Sync + 'static> {
+					self
+				}
+				#[allow(clippy::all)]
+				fn run_signal(
+					&self,
+					_calling_client: std::sync::Arc<crate::core::client::Client>,
+					_node: std::sync::Arc<crate::nodes::Node>,
+					_signal: u64,
+					_message: crate::nodes::Message
+				) -> Result<(), stardust_xr::scenegraph::ScenegraphError> {
+					match _signal {
+						#run_signals
+						_ => Err(stardust_xr::scenegraph::ScenegraphError::MemberNotFound)
+					}
+				}
+				#[allow(clippy::all)]
+				fn run_method(
+					&self,
+					_calling_client: std::sync::Arc<crate::core::client::Client>,
+					_node: std::sync::Arc<crate::nodes::Node>,
+					_method: u64,
+					_message: crate::nodes::Message,
+					_method_response: crate::nodes::MethodResponseSender,
+				) {
+					match _method {
+						#run_methods
+						_ => {
+							let _ = _method_response.send(Err(stardust_xr::scenegraph::ScenegraphError::MemberNotFound));
+						}
+					}
+				}
+			};
+		}
+	};
+	quote!(#opcodes #alias_info #client_side_members #server_side_members #aspect_macro)
+}
+
+fn generate_alias_opcodes(aspect: &Aspect, side: Side, _type: MemberType) -> TokenStream {
+	aspect
+		.members
+		.iter()
+		.filter(|m| m.side == side)
+		.filter(|m| m._type == _type)
+		.map(|m| m.opcode)
+		.map(|o| quote!(#o))
+		.reduce(|a, b| quote!(#a, #b))
+		.unwrap_or_default()
+}
+fn generate_alias_info(aspect: &Aspect) -> TokenStream {
+	let aspect_alias_info_name = Ident::new(
+		&format!(
+			"{}_ASPECT_ALIAS_INFO",
+			aspect.name.to_case(Case::ScreamingSnake)
+		),
+		Span::call_site(),
+	);
+	let local_signals = generate_alias_opcodes(aspect, Side::Server, MemberType::Signal);
+	let local_methods = generate_alias_opcodes(aspect, Side::Server, MemberType::Method);
+	let remote_signals = generate_alias_opcodes(aspect, Side::Client, MemberType::Signal);
+
+	let inherits = aspect
+		.inherits
+		.iter()
+		.map(|a| {
+			Ident::new(
+				&format!("{}_ASPECT_ALIAS_INFO", a.to_case(Case::ScreamingSnake)),
+				Span::call_site(),
+			)
+		})
+		.map(|a| quote!(#a.clone()))
+		.fold(quote!(), |a, b| quote!(#a + #b));
+
+	quote! {
+		lazy_static::lazy_static! {
+			#[allow(clippy::all)]
+			pub static ref #aspect_alias_info_name: crate::nodes::alias::AliasInfo = crate::nodes::alias::AliasInfo {
+				server_signals: vec![#local_signals],
+				server_methods: vec![#local_methods],
+				client_signals: vec![#remote_signals],
+			}
+			#inherits;
+		}
+	}
+}
+
+fn generate_member(aspect_id: u64, aspect_name: &str, member: &Member) -> TokenStream {
+	let opcode = member.opcode;
+	let name = Ident::new(&member.name.to_case(Case::Snake), Span::call_site());
+	let description = &member.description;
+
+	let side = member.side;
+	let _type = member._type;
+
+	let first_args = match member.side {
+		Side::Server => {
+			quote!(_node: std::sync::Arc<crate::nodes::Node>, _calling_client: std::sync::Arc<crate::core::client::Client>)
+		}
+		Side::Client => quote!(_node: &crate::nodes::Node),
+	};
+	let argument_decls = member
+		.arguments
+		.iter()
+		.map(|a| generate_argument_decl(a, member.side == Side::Server))
+		.fold(first_args, |a, b| quote!(#a, #b));
+
+	let argument_uses = member
+		.arguments
+		.iter()
+		.map(|a| generate_argument_serialize(&a.name, &a._type, a.optional))
+		.reduce(|a, b| quote!(#a, #b))
+		.unwrap_or_default();
+	let return_type = member
+		.return_type
+		.as_ref()
+		.map(|r| generate_argument_type(r, false, true))
+		.unwrap_or_else(|| quote!(()));
+	let name_str = name.to_string();
+
+	match (side, _type) {
+		(Side::Client, MemberType::Signal) => {
+			quote! {
+				#[doc = #description]
+				pub fn #name(#argument_decls) -> crate::core::error::Result<()> {
+
+					let result = stardust_xr::schemas::flex::serialize((#argument_uses)).map_err(|e|e.into()).and_then(|serialized|_node.send_remote_signal(#aspect_id, #opcode, serialized));
+					if let Err(err) = result.as_ref() {
+						::tracing::warn!("failed to send remote signal: {}::{}, error: {}",#aspect_name,#name_str,err);
+					} else {
+						::tracing::trace!("sent remote signal: {}::{}",#aspect_name,#name_str);
+					}
+					result
+				}
+			}
+		}
+		(Side::Client, MemberType::Method) => {
+			quote! {
+				#[doc = #description]
+				pub async fn #name(#argument_decls) -> crate::core::error::Result<(#return_type, Vec<std::os::fd::OwnedFd>)> {
+					let result = _node.execute_remote_method_typed(#aspect_id, #opcode, &(#argument_uses), vec![]).await;
+					if let Err(err) = result.as_ref() {
+						::tracing::warn!("failed to call remote method: {}::{}, error: {}",#aspect_name,#name_str,err);
+					} else {
+						::tracing::trace!("called remote method: {}::{}",#aspect_name,#name_str);
+					}
+					result
+				}
+			}
+		}
+		(Side::Server, MemberType::Signal) => {
+			quote! {
+				#[doc = #description]
+				fn #name(#argument_decls) -> crate::core::error::Result<()>;
+			}
+		}
+		(Side::Server, MemberType::Method) => {
+			quote! {
+				#[doc = #description]
+				fn #name(#argument_decls) -> impl std::future::Future<Output = crate::core::error::Result<#return_type>> + Send + Sync + 'static;
+			}
+		}
+	}
+}
+fn generate_run_member(aspect_name: &Ident, _type: MemberType, member: &Member) -> TokenStream {
+	let opcode = member.opcode;
+	let member_name_ident = Ident::new(&member.name, Span::call_site());
+
+	let argument_names = member
+		.arguments
+		.iter()
+		.map(generate_argument_name)
+		.reduce(|a, b| quote!(#a, #b));
+	let argument_types = member
+		.arguments
+		.iter()
+		.map(|a| {
+			let _type = convert_deserializeable_argument_type(&a._type);
+			generate_argument_type(&_type, a.optional, true)
+		})
+		.reduce(|a, b| quote!(#a, #b));
+	// dbg!(&argument_types);
+	let deserialize = argument_names
+		.clone()
+		.zip(argument_types)
+		.map(|(argument_names, argument_types)| {
+			quote!{
+				#[allow(unused_parens)]
+				let (#argument_names): (#argument_types) = stardust_xr::schemas::flex::deserialize(_message.as_ref())?;
+			}
+		})
+		.unwrap_or_default();
+	let serialize = generate_argument_serialize(
+		"result",
+		&member.return_type.clone().unwrap_or(ArgumentType::Empty),
+		false,
+	);
+	let argument_uses = member
+		.arguments
+		.iter()
+		.map(|a| generate_argument_deserialize(&a.name, &a._type, a.optional))
+		.reduce(|a, b| quote!(#a, #b))
+		.unwrap_or_default();
+	let member_name = member_name_ident.to_string();
+	let aspect_name_str = aspect_name.to_string();
+	match _type {
+		MemberType::Signal => quote! {
+			#opcode => { let result = (move || {
+				#deserialize
+				<Self as #aspect_name>::#member_name_ident(_node, _calling_client.clone(), #argument_uses)
+			})().map_err(|e: crate::core::error::ServerError| stardust_xr::scenegraph::ScenegraphError::MemberError { error: e.to_string() });
+				if let Err(err) = result.as_ref() {
+					::tracing::warn!("failed to receive local signal: {}::{}, error: {}",#aspect_name_str,#member_name,err);
+				} else {
+					::tracing::trace!("received local signal: {}::{}",#aspect_name_str,#member_name);
+				}
+				result
+			},
+		},
+		MemberType::Method => quote! {
+			#opcode => _method_response.wrap_async(async move {
+					#deserialize
+					let result = <Self as #aspect_name>::#member_name_ident(_node, _calling_client.clone(), #argument_uses).await;
+					if let Err(err) = result.as_ref() {
+						::tracing::warn!("failed to call local method: {}::{}, error: {}",#aspect_name_str,#member_name,err);
+					} else {
+						::tracing::trace!("called local method: {}::{}",#aspect_name_str,#member_name);
+					};
+					let result = result?;
+					 Ok((#serialize, Vec::<std::os::fd::OwnedFd>::new()))
+			}),
+		},
+	}
+}
+fn generate_argument_name(argument: &Argument) -> TokenStream {
+	Ident::new(&argument.name.to_case(Case::Snake), Span::call_site()).to_token_stream()
+}
+
+fn convert_deserializeable_argument_type(argument_type: &ArgumentType) -> ArgumentType {
+	match argument_type {
+		ArgumentType::Node { .. } => ArgumentType::NodeID,
+		ArgumentType::Vec(v) => {
+			ArgumentType::Vec(Box::new(convert_deserializeable_argument_type(v.as_ref())))
+		}
+		ArgumentType::Map(v) => {
+			ArgumentType::Map(Box::new(convert_deserializeable_argument_type(v.as_ref())))
+		}
+		f => f.clone(),
+	}
+}
+fn generate_argument_deserialize(
+	argument_name: &str,
+	argument_type: &ArgumentType,
+	optional: bool,
+) -> TokenStream {
+	let name = Ident::new(&argument_name.to_case(Case::Snake), Span::call_site());
+	if let ArgumentType::Node { .. } = argument_type {
+		return match optional {
+			true => quote!(#name.map(|n| _calling_client.get_node(#argument_name, n)?)),
+			false => quote!(_calling_client.get_node(#argument_name, #name)?),
+		};
+	}
+	if optional {
+		let mapping = generate_argument_deserialize("o", argument_type, false);
+		return quote!(#name.map(|o| Ok::<_, crate::core::error::ServerError>(#mapping)).transpose()?);
+	}
+
+	match argument_type {
+		ArgumentType::Color => quote!(color::rgba_linear!(#name[0], #name[1], #name[2], #name[3])),
+		ArgumentType::Vec(v) => {
+			let mapping = generate_argument_deserialize("a", v, false);
+			quote!(#name.into_iter().map(|a| Ok(#mapping)).collect::<crate::core::error::Result<Vec<_>>>()?)
+		}
+		ArgumentType::Map(v) => {
+			let mapping = generate_argument_deserialize("a", v, false);
+			quote!(#name.into_iter().map(|(k, a)| Ok((k, #mapping))).collect::<crate::core::error::Result<rustc_hash::FxHashMap<String, _>>>()?)
+		}
+		_ => quote!(#name),
+	}
+}
+fn generate_argument_serialize(
+	argument_name: &str,
+	argument_type: &ArgumentType,
+	optional: bool,
+) -> TokenStream {
+	let name = Ident::new(&argument_name.to_case(Case::Snake), Span::call_site());
+	match argument_type {
+		ArgumentType::Node {
+			_type,
+			return_id_parameter_name: _,
+		} => match optional {
+			true => quote!(#name.map(|n| n.get_id())),
+			false => quote!(#name.get_id()),
+		},
+		ArgumentType::Color => quote!([#name.c.r, #name.c.g, #name.c.b, #name.a]),
+		ArgumentType::Vec(v) => {
+			let mapping = generate_argument_serialize("a", v, false);
+			quote!(#name.into_iter().map(|a| Ok(#mapping)).collect::<crate::core::error::Result<Vec<_>>>()?)
+		}
+		ArgumentType::Map(v) => {
+			let mapping = generate_argument_serialize("a", v, false);
+			quote!(#name.into_iter().map(|(k, a)| Ok((k, #mapping))).collect::<crate::core::error::Result<rustc_hash::FxHashMap<String, _>>>()?)
+		}
+		_ => quote!(#name),
+	}
+}
+fn generate_argument_decl(argument: &Argument, owned_values: bool) -> TokenStream {
+	let name = Ident::new(&argument.name.to_case(Case::Snake), Span::call_site());
+	let mut _type = generate_argument_type(&argument._type, argument.optional, owned_values);
+	quote!(#name: #_type)
+}
+fn argument_type_option_name(argument_type: &ArgumentType) -> String {
+	match argument_type {
+		ArgumentType::Empty => "Empty".to_string(),
+		ArgumentType::Bool => "Bool".to_string(),
+		ArgumentType::Int => "Int".to_string(),
+		ArgumentType::UInt => "UInt".to_string(),
+		ArgumentType::Float => "Float".to_string(),
+		ArgumentType::Vec2(_) => "Vec2".to_string(),
+		ArgumentType::Vec3(_) => "Vec3".to_string(),
+		ArgumentType::Quat => "Quat".to_string(),
+		ArgumentType::Mat4 => "Mat4".to_string(),
+		ArgumentType::Color => "Color".to_string(),
+		ArgumentType::String => "String".to_string(),
+		ArgumentType::Bytes => "Bytes".to_string(),
+		ArgumentType::Vec(v) => format!("{}Vector", argument_type_option_name(v)),
+		ArgumentType::Map(m) => format!("{}Map", argument_type_option_name(m)),
+		ArgumentType::NodeID => "Node ID".to_string(),
+		ArgumentType::Datamap => "Datamap".to_string(),
+		ArgumentType::ResourceID => "ResourceID".to_string(),
+		ArgumentType::Enum(e) => e.clone(),
+		ArgumentType::Union(u) => u.clone(),
+		ArgumentType::Struct(s) => s.clone(),
+		ArgumentType::Node { _type, .. } => _type.clone(),
+		ArgumentType::Fd => "File Descriptor".to_string(),
+	}
+}
+fn generate_argument_type(
+	argument_type: &ArgumentType,
+	optional: bool,
+	owned: bool,
+) -> TokenStream {
+	let _type = match argument_type {
+		ArgumentType::Empty => quote!(()),
+		ArgumentType::Bool => quote!(bool),
+		ArgumentType::Int => quote!(i32),
+		ArgumentType::UInt => quote!(u32),
+		ArgumentType::Float => quote!(f32),
+		ArgumentType::Vec2(t) => {
+			let t = generate_argument_type(t, false, true);
+			quote!(stardust_xr::values::Vector2<#t>)
+		}
+		ArgumentType::Vec3(t) => {
+			let t = generate_argument_type(t, false, true);
+			quote!(stardust_xr::values::Vector3<#t>)
+		}
+		ArgumentType::Quat => quote!(stardust_xr::values::Quaternion),
+		ArgumentType::Mat4 => quote!(stardust_xr::values::Mat4),
+		ArgumentType::Color => quote!(stardust_xr::values::Color),
+		ArgumentType::Bytes => {
+			if !owned {
+				quote!(&[u8])
+			} else {
+				quote!(Vec<u8>)
+			}
+		}
+		ArgumentType::String => {
+			if !owned {
+				quote!(&str)
+			} else {
+				quote!(String)
+			}
+		}
+		ArgumentType::Vec(t) => {
+			let t = generate_argument_type(t, false, true);
+			if !owned {
+				quote!(&[#t])
+			} else {
+				quote!(Vec<#t>)
+			}
+		}
+		ArgumentType::Map(t) => {
+			let t = generate_argument_type(t, false, true);
+
+			if !owned {
+				quote!(&stardust_xr::values::Map<String, #t>)
+			} else {
+				quote!(stardust_xr::values::Map<String, #t>)
+			}
+		}
+		ArgumentType::NodeID => quote!(u64),
+		ArgumentType::Datamap => {
+			if !owned {
+				quote!(&stardust_xr::values::Datamap)
+			} else {
+				quote!(stardust_xr::values::Datamap)
+			}
+		}
+		ArgumentType::ResourceID => {
+			if !owned {
+				quote!(&stardust_xr::values::ResourceID)
+			} else {
+				quote!(stardust_xr::values::ResourceID)
+			}
+		}
+		ArgumentType::Enum(e) => {
+			let enum_name = Ident::new(&e.to_case(Case::Pascal), Span::call_site());
+			quote!(#enum_name)
+		}
+		ArgumentType::Union(u) => {
+			let union_name = Ident::new(&u.to_case(Case::Pascal), Span::call_site());
+			quote!(#union_name)
+		}
+		ArgumentType::Struct(s) => {
+			let struct_name = Ident::new(&s.to_case(Case::Pascal), Span::call_site());
+			if !owned {
+				quote!(&#struct_name)
+			} else {
+				quote!(#struct_name)
+			}
+		}
+		ArgumentType::Node {
+			_type,
+			return_id_parameter_name: _,
+		} => {
+			if !owned {
+				quote!(&std::sync::Arc<crate::nodes::Node>)
+			} else {
+				quote!(std::sync::Arc<crate::nodes::Node>)
+			}
+		}
+		ArgumentType::Fd => {
+			quote!(&std::os::fd::OwnedFd)
+		}
+	};
+
+	if optional {
+		quote!(Option<#_type>)
+	} else {
+		_type
+	}
+}

flake.lock

@@ -0,0 +1,171 @@
+{
+  "nodes": {
+    "crane": {
+      "inputs": {
+        "nixpkgs": [
+          "flatland",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1712681629,
+        "narHash": "sha256-bMDXn4AkTXLCpoZbII6pDGoSeSe9gI87jxPsHRXgu/E=",
+        "owner": "ipetkov",
+        "repo": "crane",
+        "rev": "220387ac8e99cbee0ca4c95b621c4bc782b6a235",
+        "type": "github"
+      },
+      "original": {
+        "owner": "ipetkov",
+        "repo": "crane",
+        "type": "github"
+      }
+    },
+    "flake-parts": {
+      "inputs": {
+        "nixpkgs-lib": "nixpkgs-lib"
+      },
+      "locked": {
+        "lastModified": 1722555600,
+        "narHash": "sha256-XOQkdLafnb/p9ij77byFQjDf5m5QYl9b2REiVClC+x4=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "8471fe90ad337a8074e957b69ca4d0089218391d",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "type": "github"
+      }
+    },
+    "flake-parts_2": {
+      "inputs": {
+        "nixpkgs-lib": [
+          "hercules-ci-effects",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1712014858,
+        "narHash": "sha256-sB4SWl2lX95bExY2gMFG5HIzvva5AVMJd4Igm+GpZNw=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "9126214d0a59633752a136528f5f3b9aa8565b7d",
+        "type": "github"
+      },
+      "original": {
+        "id": "flake-parts",
+        "type": "indirect"
+      }
+    },
+    "flatland": {
+      "inputs": {
+        "crane": "crane",
+        "nixpkgs": "nixpkgs"
+      },
+      "locked": {
+        "lastModified": 1713050562,
+        "narHash": "sha256-m7c6XpmpTM1URuqMG2KqtaWbL2Vt8vJFJtmvq123BmY=",
+        "owner": "StardustXR",
+        "repo": "flatland",
+        "rev": "b3b0f29c4ea1b82c96cf9de507837bf15a5e4c0e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "StardustXR",
+        "repo": "flatland",
+        "type": "github"
+      }
+    },
+    "hercules-ci-effects": {
+      "inputs": {
+        "flake-parts": "flake-parts_2",
+        "nixpkgs": "nixpkgs_2"
+      },
+      "locked": {
+        "lastModified": 1719226092,
+        "narHash": "sha256-YNkUMcCUCpnULp40g+svYsaH1RbSEj6s4WdZY/SHe38=",
+        "owner": "hercules-ci",
+        "repo": "hercules-ci-effects",
+        "rev": "11e4b8dc112e2f485d7c97e1cee77f9958f498f5",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "hercules-ci-effects",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1712791164,
+        "narHash": "sha256-3sbWO1mbpWsLepZGbWaMovSO7ndZeFqDSdX0hZ9nVyw=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "1042fd8b148a9105f3c0aca3a6177fd1d9360ba5",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs-lib": {
+      "locked": {
+        "lastModified": 1722555339,
+        "narHash": "sha256-uFf2QeW7eAHlYXuDktm9c25OxOyCoUOQmh5SZ9amE5Q=",
+        "type": "tarball",
+        "url": "https://github.com/NixOS/nixpkgs/archive/a5d394176e64ab29c852d03346c1fc9b0b7d33eb.tar.gz"
+      },
+      "original": {
+        "type": "tarball",
+        "url": "https://github.com/NixOS/nixpkgs/archive/a5d394176e64ab29c852d03346c1fc9b0b7d33eb.tar.gz"
+      }
+    },
+    "nixpkgs_2": {
+      "locked": {
+        "lastModified": 1713714899,
+        "narHash": "sha256-+z/XjO3QJs5rLE5UOf015gdVauVRQd2vZtsFkaXBq2Y=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "6143fc5eeb9c4f00163267708e26191d1e918932",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "nixpkgs_3": {
+      "locked": {
+        "lastModified": 1723991338,
+        "narHash": "sha256-Grh5PF0+gootJfOJFenTTxDTYPidA3V28dqJ/WV7iis=",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "8a3354191c0d7144db9756a74755672387b702ba",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nixos",
+        "ref": "nixos-unstable",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "flake-parts": "flake-parts",
+        "flatland": "flatland",
+        "hercules-ci-effects": "hercules-ci-effects",
+        "nixpkgs": "nixpkgs_3"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

flake.nix

@@ -0,0 +1,120 @@
+{
+  nixConfig = {
+    extra-substituters = [ "https://stardustxr.cachix.org" ];
+    extra-trusted-public-keys = [
+      "stardustxr.cachix.org-1:mWSn8Ap2RLsIWT/8gsj+VfbJB6xoOkPaZpbjO+r9HBo="
+    ];
+  };
+  inputs = {
+    nixpkgs.url = "github:nixos/nixpkgs/nixos-unstable";
+    flake-parts.url = "github:hercules-ci/flake-parts";
+    hercules-ci-effects.url = "github:hercules-ci/hercules-ci-effects";
+
+    # Since we do not have a monorepo, we have to fetch Flatland in order to use
+    # it to create VM Tests
+    flatland.url = "github:StardustXR/flatland";
+  };
+  outputs =
+    inputs@{
+      self,
+      flake-parts,
+      nixpkgs,
+      hercules-ci-effects,
+      flatland,
+      ...
+    }:
+    let
+      name = (builtins.fromTOML (builtins.readFile ./Cargo.toml)).package.name;
+      src = builtins.path {
+        name = "${name}-source";
+        path = toString ./.;
+        filter =
+          path: type:
+          nixpkgs.lib.all (n: builtins.baseNameOf path != n) [
+            "flake.nix"
+            "flake.lock"
+            "nix"
+            "README.md"
+          ];
+      };
+    in
+    flake-parts.lib.mkFlake { inherit inputs; } {
+      imports = [ flake-parts.flakeModules.easyOverlay ];
+      systems = [
+        "aarch64-linux"
+        "x86_64-linux"
+        "riscv64-linux"
+      ];
+      perSystem =
+        {
+          config,
+          self',
+          inputs',
+          pkgs,
+          system,
+          ...
+        }:
+        {
+          _module.args.pkgs = import inputs.nixpkgs {
+            inherit system;
+            overlays = [ inputs.self.overlays.default ];
+          };
+          overlayAttrs = config.packages;
+          packages =
+            let
+              sk_gpu = pkgs.callPackage ./nix/sk_gpu.nix { };
+            in
+            {
+              default = self'.packages.${name};
+              gnome-graphical-test = self'.checks.gnome-graphical-test;
+              "${name}" = pkgs.callPackage ./nix/stardust-xr-server.nix {
+                inherit name src sk_gpu;
+              };
+            };
+          apps.default = {
+            type = "app";
+            program = self'.packages.${name} + "/bin/stardust-xr-server";
+          };
+          checks.gnome-graphical-test = pkgs.nixosTest (
+            import ./nix/gnome-graphical-test.nix { inherit pkgs self; }
+          );
+          devShells.default = pkgs.mkShell {
+            inputsFrom = [ self'.packages.default ];
+            LIBCLANG_PATH = "${pkgs.libclang.lib}/lib";
+          };
+        };
+      flake = {
+        herculesCI.ciSystems = [ "x86_64-linux" ];
+        effects =
+          let
+            pkgs = nixpkgs.legacyPackages.x86_64-linux;
+            hci-effects = hercules-ci-effects.lib.withPkgs pkgs;
+          in
+          { ref, rev, ... }:
+          {
+            gnome-graphical-test = hci-effects.mkEffect {
+              secretsMap."stardustxrDiscord" = "stardustxrDiscord";
+              secretsMap."stardustxrIpfs" = "stardustxrIpfs";
+              effectScript = ''
+                readSecretString stardustxrDiscord .webhook > .webhook
+                readSecretString stardustxrIpfs .basicauth > .basicauth
+                set -x
+                export RESPONSE=$(curl -H @.basicauth -F file=@${
+                  self.packages."x86_64-linux".gnome-graphical-test
+                }/screen.png https://ipfs-api.stardustxr.org/api/v0/add)
+                export CID=$(echo "$RESPONSE" | ${pkgs.jq}/bin/jq -r .Hash)
+                set +x
+                export ADDRESS="https://ipfs.stardustxr.org/ipfs/$CID"
+                ${pkgs.discord-sh}/bin/discord.sh \
+                  --description "\`stardustxr/server\` has been modified, here's how it renders \`weston-cliptest\` on \`flatland\` via \`monado-service\` inside of the \`gnome-graphical-test\`" \
+                  --field "Ref;${ref}" \
+                  --field "Commit ID;${rev}" \
+                  --field "Flatland Revision;${flatland.rev}" \
+                  --field "Reproducer;\`nix build github:stardustxr/server/${rev}#gnome-graphical-test\`" \
+                  --image "$ADDRESS"
+              '';
+            };
+          };
+      };
+    };
+}

justfile

@@ -0,0 +1,12 @@
+PREFIX := "usr"
+BINARY := PREFIX / "bin"
+DESTDIR := "/"
+
+build:
+    cargo build --release
+
+test:
+	cargo test
+
+install:
+    install -Dm755 target/release/stardust-xr-server {{ DESTDIR }}{{ BINARY }}/stardust-xr-server

nix/gnome-graphical-test.nix

@@ -0,0 +1,155 @@
+{ pkgs, lib ? pkgs.lib, self, ...  }:
+# Some code is copy-pasted from https://github.com/NixOS/nixpkgs/blob/master/nixos/tests/gnome.nix
+# TODO: make this less boiler-platey and make a function like mkGnomeTest that does all this and upstream it to nixpkgs
+{
+  name = "stardust-xr-server-gnome-vmtest";
+  meta = with lib; {
+    maintainers = [ maintainers.matthewcroughan ];
+  };
+  nodes.machine = { ... }: {
+    imports = [ "${pkgs.path}/nixos/tests/common/user-account.nix" ];
+    virtualisation.qemu.options = [
+      "-device virtio-gpu-pci"
+    ];
+    environment.systemPackages = [ pkgs.monado ];
+    services.xserver = {
+      enable = true;
+      desktopManager.gnome = {
+        enable = true;
+        debug = true;
+        # Set a nice desktop background that is pleasing to the eyes :3
+        extraGSettingsOverrides = ''
+          [org.gnome.desktop.background]
+          picture-uri='file://${pkgs.gnome.gnome-backgrounds}/share/backgrounds/gnome/blobs-l.svg'
+          picture-uri-dark='file://${pkgs.gnome.gnome-backgrounds}/share/backgrounds/gnome/blobs-l.svg'
+        '';
+      };
+      displayManager = {
+        gdm = {
+          enable = true;
+          debug = true;
+        };
+        autoLogin = {
+          enable = true;
+          user = "alice";
+        };
+      };
+    };
+    systemd.user.services = {
+      "monado" = {
+        after = [ "graphical-session.target" "default.target" "org.gnome.Shell@wayland.service" ];
+        environment = {
+          XRT_COMPOSITOR_FORCE_WAYLAND = "1";
+          WAYLAND_DISPLAY = "wayland-0";
+        };
+        serviceConfig = {
+          ExecStartPre = [
+            "${pkgs.writeShellScript "sleep" ''
+              sleep 3
+            ''}"
+          ];
+          ExecStart = let
+            # stdin disappears in NixOS test driver ( machine.succeed() ), requiring us to specify < /dev/ttyS0 to fake stdin
+            exec-monado-service = pkgs.writeShellScript "exec-monado-service" "${pkgs.monado}/bin/monado-service < /dev/ttyS0";
+          in [
+            "${exec-monado-service}"
+          ];
+        };
+      };
+      "stardust-xr-server" = {
+        after = [ "monado.service" ];
+        serviceConfig = {
+          Type = "notify";
+          NotifyAccess = "all";
+          ExecStartPre = [
+            "${pkgs.writeShellScript "sleep" ''
+              sleep 3
+            ''}"
+          ];
+          ExecStart = let
+            notifyReady = pkgs.writeShellScript "notifyReady" "systemd-notify --ready";
+            exec-stardust-xr-server = pkgs.writeShellScript "exec-stardust-xr-server" "${self.packages.${pkgs.hostPlatform.system}.default}/bin/stardust-xr-server -e ${notifyReady}";
+          in [
+            "${exec-stardust-xr-server}"
+          ];
+        };
+      };
+      "weston-cliptest" = {
+        after = [ "flatland.service" ];
+        environment.WAYLAND_DISPLAY = "wayland-1";
+        serviceConfig = {
+          ExecStart = [
+            "${pkgs.weston}/bin/weston-cliptest"
+          ];
+        };
+      };
+      "flatland" = {
+        after = [ "stardust-xr-server.service" ];
+        serviceConfig = {
+          ExecStart = [
+            "${self.inputs.flatland.packages.${pkgs.hostPlatform.system}.default}/bin/flatland"
+          ];
+        };
+      };
+      "org.gnome.Shell@wayland" = {
+        wants = [ "monado.service" "stardust-xr-server.service" "flatland.service" "weston-cliptest.service" ];
+        serviceConfig = {
+          ExecStart = [
+            # Clear the list before overriding it.
+            ""
+            # Eval API is now internal so Shell needs to run in unsafe mode.
+            # TODO: improve test driver so that it supports openqa-like manipulation
+            # that would allow us to drop this mess.
+            "${pkgs.gnome.gnome-shell}/bin/gnome-shell --unsafe-mode"
+          ];
+        };
+      };
+    };
+  };
+
+  testScript = { nodes, ... }: let
+    # Keep line widths somewhat managable
+    user = nodes.machine.config.users.users.alice;
+    uid = toString user.uid;
+    bus = "DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/${uid}/bus";
+    gdbus = "${bus} gdbus";
+    su = command: "su ${user.name} -c '${command}'";
+
+    # Call javascript in gnome shell, returns a tuple (success, output), where
+    # `success` is true if the dbus call was successful and output is what the
+    # javascript evaluates to.
+    eval = "call --session -d org.gnome.Shell -o /org/gnome/Shell -m org.gnome.Shell.Eval";
+
+    # False when startup is done
+    startingUp = su "${gdbus} ${eval} Main.layoutManager._startingUp";
+  in ''
+      with subtest("Login to GNOME with GDM"):
+          # wait for gdm to start
+          machine.wait_for_unit("display-manager.service")
+          # wait for the wayland server
+          machine.wait_for_file("/run/user/${uid}/wayland-0")
+          # wait for alice to be logged in
+          machine.wait_for_unit("default.target", "${user.name}")
+          # check that logging in has given the user ownership of devices
+          assert "alice" in machine.succeed("getfacl -p /dev/snd/timer")
+
+      with subtest("Wait for GNOME Shell"):
+          # correct output should be (true, 'false')
+          machine.wait_until_succeeds(
+              "${startingUp} | grep -q 'true,..false'"
+          )
+
+      # To allow monado-service to use < /dev/ttyS0
+      machine.succeed("chown alice /dev/ttyS0")
+
+      with subtest("Open Monado and StardustXR"):
+          # Close the Activities view so that Shell can correctly track the focused window.
+          machine.send_key("esc")
+          machine.wait_for_unit("monado.service", "${user.name}")
+          machine.wait_for_unit("stardust-xr-server.service", "${user.name}")
+          machine.wait_for_unit("flatland.service", "${user.name}")
+          machine.wait_for_unit("weston-cliptest.service", "${user.name}")
+          machine.sleep(3)
+          machine.screenshot("screen")
+    '';
+}

nix/meshoptimizer.nix

@@ -0,0 +1,27 @@
+{ lib, stdenv, fetchFromGitHub, cmake }:
+
+stdenv.mkDerivation rec {
+  pname = "meshoptimizer";
+  version = "0.20";
+
+  src = fetchFromGitHub {
+    owner = "zeux";
+    repo = "meshoptimizer";
+    rev = "c21d3be6ddf627f8ca852ba4b6db9903b0557858";
+    sha256 = "sha256-QCxpM2g8WtYSZHkBzLTJNQ/oHb5j/n9rjaVmZJcCZIA=";
+  };
+
+  nativeBuildInputs = [ cmake ];
+
+  cmakeFlags = [
+    "-DCMAKE_POSITION_INDEPENDENT_CODE=ON"
+  ];
+
+  meta = with lib; {
+    description = "Mesh optimization library that makes meshes smaller and faster to render";
+    homepage = "https://github.com/zeux/meshoptimizer";
+    license = licenses.mit;
+    platforms = platforms.all;
+  };
+}
+  

nix/sk_gpu.nix

@@ -0,0 +1,16 @@
+{ stdenv, fetchurl, unzip }:
+
+let
+  sk_gpu_zip = fetchurl {
+    url =
+      "https://github.com/StereoKit/sk_gpu/releases/download/v2024.8.16/sk_gpu.v2024.8.16.zip";
+    sha256 = "sha256-Wk3PZFlWqhrsQ8xG0sQaV2xSasdg2D7TMiPvl/CgtGU=";
+  };
+in stdenv.mkDerivation rec {
+  name = "sk_gpu";
+  src = sk_gpu_zip;
+  unpackPhase = ''
+    unzip -d $out ${sk_gpu_zip}
+  '';
+  nativeBuildInputs = [ unzip ];
+}

nix/stardust-xr-server.nix

@@ -0,0 +1,86 @@
+{ rustPlatform
+, src
+, name
+, libGL
+, mesa
+, xorg
+, fontconfig
+, libxkbcommon
+, libclang
+
+, cmake
+, cpm-cmake
+, pkg-config
+, llvmPackages
+, fetchFromGitHub
+, sk_gpu
+, libXau
+
+, libXdmcp
+, stdenv
+, lib
+, openxr-loader
+}:
+
+rustPlatform.buildRustPackage rec {
+  inherit src name;
+  cargoLock = {
+    lockFile = (src + "/Cargo.lock");
+    allowBuiltinFetchGit = true;
+  };
+  buildFeatures = [ "local_deps" ];
+  FORCE_LOCAL_DEPS = true;
+  CPM_LOCAL_PACKAGES_ONLY = true;
+  CPM_SOURCE_CACHE = "./build";
+  CPM_USE_LOCAL_PACKAGES = true;
+  CPM_DOWNLOAD_ALL = false;
+
+  meshoptimizer = fetchFromGitHub {
+    owner = "zeux";
+    repo = "meshoptimizer";
+    rev = "c21d3be6ddf627f8ca852ba4b6db9903b0557858";
+    sha256 = "sha256-QCxpM2g8WtYSZHkBzLTJNQ/oHb5j/n9rjaVmZJcCZIA=";
+  };
+  basis_universal = fetchFromGitHub {
+    owner = "BinomialLLC";
+    repo = "basis_universal";
+    rev = "900e40fb5d2502927360fe2f31762bdbb624455f";
+    sha256 = "sha256-zBRAXgG5Fi6+5uPQCI/RCGatY6O4ELuYBoKrPNn4K+8=";
+  };
+
+  DEP_MESHOPTIMIZER_SOURCE = "${meshoptimizer}";
+  DEP_BASIS_UNIVERSAL_SOURCE = "${basis_universal}";
+  DEP_SK_GPU_SOURCE = "${sk_gpu}";
+
+  postPatch = let libPath = lib.makeLibraryPath [ stdenv.cc.cc.lib ];
+  in ''
+    sk=$(echo $cargoDepsCopy/stereokit-rust-*/StereoKit)
+    mkdir -p $sk/build/cpm
+
+    # This is not ideal, the original approach was to fetch the exact cmake
+    # file version that was wanted from GitHub directly, but at least this way it comes from Nixpkgs.. so meh
+    cp ${cpm-cmake}/share/cpm/CPM.cmake $sk/build/cpm/CPM_0.38.7.cmake
+    mkdir -p $sk/sk_gpu
+    cp -R ${sk_gpu}/* $sk/sk_gpu
+    chmod -R 755 $sk/sk_gpu/tools/linux_x64/*
+    export DEP_SK_GPU_SOURCE=$sk/sk_gpu
+    export LD_LIBRARY_PATH="${stdenv.cc.cc.lib}/lib";
+                    patchelf --set-interpreter "$(cat $NIX_CC/nix-support/dynamic-linker)" \
+                      --set-rpath "${libPath}" \
+                    $sk/sk_gpu/tools/linux_x64/skshaderc
+  '';
+  nativeBuildInputs = [ cmake pkg-config llvmPackages.libcxxClang ];
+  buildInputs = [
+    libGL
+    mesa
+    xorg.libX11.dev
+    xorg.libXft
+    xorg.libXfixes
+    fontconfig
+    libxkbcommon
+    libXau
+    libXdmcp
+    openxr-loader
+  ];
+  LIBCLANG_PATH = "${libclang.lib}/lib";
+}

src/core/bevy_channel.rs

@@ -0,0 +1,31 @@
+use std::sync::OnceLock;
+
+use bevy::prelude::*;
+use tokio::sync::mpsc::{self, error::TryRecvError};
+
+#[derive(Resource)]
+pub struct BevyChannelReader<T: Send + Sync + 'static>(mpsc::UnboundedReceiver<T>);
+pub struct BevyChannel<T: Send + Sync + 'static>(OnceLock<mpsc::UnboundedSender<T>>);
+impl<T: Send + Sync + 'static> BevyChannel<T> {
+	pub const fn new() -> Self {
+		Self(OnceLock::new())
+	}
+	pub fn init(&self, app: &mut App) {
+		let (tx, rx) = mpsc::unbounded_channel();
+		self.0.set(tx).unwrap();
+		app.insert_resource(BevyChannelReader(rx));
+	}
+	pub fn send(&self, msg: T) -> Option<()> {
+		self.0.get()?.send(msg).ok()
+	}
+}
+
+impl<T: Send + Sync + 'static> BevyChannelReader<T> {
+	pub fn read(&mut self) -> Option<T> {
+		match self.0.try_recv() {
+			Ok(v) => Some(v),
+			Err(TryRecvError::Disconnected) => panic!("bevy channel should never disconnect"),
+			Err(TryRecvError::Empty) => None,
+		}
+	}
+}

src/core/client.rs

@@ -1,136 +1,266 @@
-use super::{eventloop::EventLoop, scenegraph::Scenegraph};
+use super::{
+	client_state::{CLIENT_STATES, ClientStateParsed},
+	scenegraph::Scenegraph,
+};
 use crate::{
-	core::registry::Registry,
-	nodes::{data, drawable, fields, hmd, input, items, root::Root, spatial, startup, Node},
+	core::{registry::OwnedRegistry, task},
+	nodes::{
+		Node, audio, drawable, fields, input, items,
+		root::{ClientState, Root},
+		spatial,
+	},
 };
-use anyhow::{anyhow, Result};
+use color_eyre::eyre::{Result, eyre};
+use global_counter::primitive::exact::CounterU32;
 use lazy_static::lazy_static;
-use once_cell::sync::OnceCell;
 use parking_lot::Mutex;
-use stardust_xr::messenger::Messenger;
+use rustc_hash::FxHashMap;
+use stardust_xr::messenger::{self, MessageSenderHandle};
 use std::{
+	fmt::Debug,
+	fs,
+	iter::FromIterator,
 	path::PathBuf,
-	sync::{Arc, Weak},
+	sync::{Arc, OnceLock},
+	time::Instant,
 };
-use tokio::{net::UnixStream, sync::Notify, task::JoinHandle};
+use tokio::{net::UnixStream, sync::watch, task::JoinHandle};
+use tracing::info;
+
+pub static CLIENTS: OwnedRegistry<Client> = OwnedRegistry::new();
 
 lazy_static! {
-	pub static ref CLIENTS: Registry<Client> = Registry::new();
+	static ref INTERNAL_CLIENT_MESSAGE_TIMES: (watch::Sender<Instant>, watch::Receiver<Instant>) = watch::channel(Instant::now());
 	pub static ref INTERNAL_CLIENT: Arc<Client> = CLIENTS.add(Client {
-		event_loop: Weak::new(),
-		index: 0,
+		pid: None,
+		// env: None,
+		exe: None,
 
-		stop_notifier: Default::default(),
-		join_handle: OnceCell::new(),
+		dispatch_join_handle: OnceLock::new(),
+		flush_join_handle: OnceLock::new(),
+		disconnect_status: OnceLock::new(),
 
-		messenger: None,
+		id_counter: CounterU32::new(0),
+		message_last_received: INTERNAL_CLIENT_MESSAGE_TIMES.1.clone(),
+		message_sender_handle: None,
 		scenegraph: Default::default(),
-		root: OnceCell::new(),
+		root: OnceLock::new(),
 		base_resource_prefixes: Default::default(),
+		state: OnceLock::default(),
 	});
 }
+pub fn tick_internal_client() {
+	let _ = INTERNAL_CLIENT_MESSAGE_TIMES.0.send(Instant::now());
+}
+
+pub fn get_env(pid: i32) -> Result<FxHashMap<String, String>, std::io::Error> {
+	let env = fs::read_to_string(format!("/proc/{pid}/environ"))?;
+	Ok(FxHashMap::from_iter(
+		env.split('\0')
+			.filter_map(|var| var.split_once('='))
+			.map(|(k, v)| (k.to_string(), v.to_string())),
+	))
+}
+pub fn state(env: &FxHashMap<String, String>) -> Option<Arc<ClientStateParsed>> {
+	let token = env.get("STARDUST_STARTUP_TOKEN")?;
+	CLIENT_STATES.lock().get(token).cloned()
+}
 
 pub struct Client {
-	event_loop: Weak<EventLoop>,
-	index: usize,
-	stop_notifier: Arc<Notify>,
-	join_handle: OnceCell<JoinHandle<Result<()>>>,
+	pub pid: Option<i32>,
+	// env: Option<FxHashMap<String, String>>,
+	exe: Option<PathBuf>,
+	dispatch_join_handle: OnceLock<JoinHandle<Result<()>>>,
+	flush_join_handle: OnceLock<JoinHandle<Result<()>>>,
+	disconnect_status: OnceLock<Result<()>>,
 
-	pub messenger: Option<Messenger>,
-	pub scenegraph: Scenegraph,
-	pub root: OnceCell<Arc<Root>>,
+	id_counter: CounterU32,
+	message_last_received: watch::Receiver<Instant>,
+	pub message_sender_handle: Option<MessageSenderHandle>,
+	pub scenegraph: Arc<Scenegraph>,
+	pub root: OnceLock<Arc<Root>>,
 	pub base_resource_prefixes: Mutex<Vec<PathBuf>>,
+	pub state: OnceLock<ClientState>,
 }
 impl Client {
-	pub fn from_connection(
-		index: usize,
-		event_loop: &Arc<EventLoop>,
-		connection: UnixStream,
-	) -> Arc<Self> {
-		println!("New client connected");
-		let client = CLIENTS.add(Client {
-			event_loop: Arc::downgrade(event_loop),
-			index,
-			stop_notifier: Default::default(),
-			join_handle: OnceCell::new(),
+	pub fn from_connection(connection: UnixStream) -> Result<Arc<Self>> {
+		let pid = connection.peer_cred().ok().and_then(|c| c.pid());
+		let env = pid.and_then(|pid| get_env(pid).ok());
+		let exe = pid.and_then(|pid| fs::read_link(format!("/proc/{pid}/exe")).ok());
+		info!(
+			pid,
+			exe = exe
+				.as_ref()
+				.and_then(|exe| exe.to_str().map(|s| s.to_string())),
+			"New client connected"
+		);
 
-			messenger: Some(Messenger::new(
-				tokio::runtime::Handle::current(),
-				connection,
-			)),
-			scenegraph: Default::default(),
-			root: OnceCell::new(),
+		let (mut messenger_tx, mut messenger_rx) = messenger::create(connection);
+		let scenegraph = Arc::new(Scenegraph::default());
+		let state = env
+			.as_ref()
+			.and_then(state)
+			.unwrap_or_else(|| Arc::new(ClientStateParsed::default()));
+
+		let (message_time_tx, message_last_received) = watch::channel(Instant::now());
+		let client = CLIENTS.add(Client {
+			pid,
+			// env,
+			exe: exe.clone(),
+
+			dispatch_join_handle: OnceLock::new(),
+			flush_join_handle: OnceLock::new(),
+			disconnect_status: OnceLock::new(),
+
+			id_counter: CounterU32::new(256),
+			message_last_received,
+			message_sender_handle: Some(messenger_tx.handle()),
+			scenegraph: scenegraph.clone(),
+			root: OnceLock::new(),
 			base_resource_prefixes: Default::default(),
+			state: OnceLock::default(),
 		});
 		let _ = client.scenegraph.client.set(Arc::downgrade(&client));
-		let _ = client.root.set(Root::create(&client));
-		hmd::make_alias(&client);
-		spatial::create_interface(&client);
-		fields::create_interface(&client);
-		drawable::create_interface(&client);
-		data::create_interface(&client);
-		items::create_interface(&client);
-		input::create_interface(&client);
-		startup::create_interface(&client);
+		let _ = client.root.set(Root::create(&client, state.root)?);
+		spatial::create_interface(&client)?;
+		fields::create_interface(&client)?;
+		drawable::create_interface(&client)?;
+		audio::create_interface(&client)?;
+		input::create_interface(&client)?;
+		items::camera::create_interface(&client)?;
+		items::panel::create_interface(&client)?;
 
-		let _ = client.join_handle.set(tokio::spawn({
-			let client = client.clone();
-			async move {
-				let dispatch_loop = async {
-					loop {
-						client.dispatch().await?
-					}
-				};
-				let flush_loop = async {
-					loop {
-						client.flush().await?
-					}
-				};
+		let _ = client.state.set(state.apply_to(&client));
 
-				let result = tokio::select! {
-					_ = client.stop_notifier.notified() => Ok(()),
-					e = dispatch_loop => e,
-					e = flush_loop => e,
-				};
-				client.disconnect().await;
-				result
-			}
-		}));
-		client
+		let pid_printable = pid
+			.map(|pid| pid.to_string())
+			.unwrap_or_else(|| "??".to_string());
+		let exe_printable = exe
+			.and_then(|exe| {
+				exe.file_name()
+					.and_then(|exe| exe.to_str())
+					.map(|exe| exe.to_string())
+			})
+			.unwrap_or_else(|| "??".to_string());
+		let _ = client.dispatch_join_handle.get_or_init(|| {
+			task::new(
+				|| {
+					format!(
+						"client dispatch pid={} exe={}",
+						&pid_printable, &exe_printable,
+					)
+				},
+				{
+					let client = client.clone();
+					async move {
+						loop {
+							if let Err(e) = messenger_rx.dispatch(&*scenegraph).await {
+								client.disconnect(Err(e.into()));
+							}
+							let _ = message_time_tx.send(Instant::now());
+						}
+					}
+				},
+			)
+			.unwrap()
+		});
+		let _ = client.flush_join_handle.get_or_init(|| {
+			task::new(
+				|| format!("client flush pid={} exe={}", &pid_printable, &exe_printable,),
+				{
+					let client = client.clone();
+					async move {
+						loop {
+							if let Err(e) = messenger_tx.flush().await {
+								client.disconnect(Err(e.into()));
+							}
+						}
+					}
+				},
+			)
+			.unwrap()
+		});
+
+		Ok(client)
+	}
+
+	pub fn get_cmdline(&self) -> Option<Vec<String>> {
+		let pid = self.pid?;
+		let exe_proc_path = format!("/proc/{pid}/exe");
+		let cmdline_proc_path = format!("/proc/{pid}/cmdline");
+		let exe = std::fs::read_link(exe_proc_path).ok()?;
+		let cmdline = std::fs::read_to_string(cmdline_proc_path).ok()?;
+		let mut cmdline_split: Vec<_> = cmdline.split('\0').map(ToString::to_string).collect();
+		cmdline_split.pop();
+		*cmdline_split.get_mut(0).unwrap() = exe.to_str()?.to_string();
+		Some(cmdline_split)
+	}
+	pub fn get_cwd(&self) -> Option<PathBuf> {
+		let pid = self.pid?;
+		let cwd_proc_path = format!("/proc/{pid}/cwd");
+		std::fs::read_link(cwd_proc_path).ok()
+	}
+	pub async fn save_state(&self) -> Option<ClientStateParsed> {
+		println!("start save state");
+		let internal = self.root.get()?.save_state().await.ok()?;
+		println!("finished save state");
+		Some(ClientStateParsed::from_deserialized(self, internal))
+	}
+
+	pub fn generate_id(&self) -> u64 {
+		self.id_counter.inc() as u64
 	}
 
 	#[inline]
-	pub fn get_node(&self, name: &'static str, path: &str) -> Result<Arc<Node>> {
+	pub fn get_node(&self, name: &'static str, id: u64) -> Result<Arc<Node>> {
 		self.scenegraph
-			.get_node(path)
-			.ok_or_else(|| anyhow!("{} not found", name))
+			.get_node(id)
+			.ok_or_else(|| eyre!("{} not found", name))
 	}
 
-	pub async fn dispatch(&self) -> Result<(), std::io::Error> {
-		match &self.messenger {
-			Some(messenger) => messenger.dispatch(&self.scenegraph).await,
-			None => Err(std::io::Error::from(std::io::ErrorKind::Unsupported)),
-		}
+	pub fn unresponsive(&self) -> bool {
+		let time_since_last_message = self.message_last_received.borrow().elapsed();
+		time_since_last_message.as_millis() > 500
 	}
 
-	pub async fn flush(&self) -> Result<(), std::io::Error> {
-		match &self.messenger {
-			Some(messenger) => messenger.flush().await,
-			None => Err(std::io::Error::from(std::io::ErrorKind::Unsupported)),
+	pub fn disconnect(&self, reason: Result<()>) {
+		let _ = self.disconnect_status.set(reason);
+		if let Some(dispatch_join_handle) = self.dispatch_join_handle.get() {
+			dispatch_join_handle.abort();
 		}
+		if let Some(flush_join_handle) = self.flush_join_handle.get() {
+			flush_join_handle.abort();
+		}
+		CLIENTS.remove(self);
 	}
-
-	pub async fn disconnect(&self) {
-		self.stop_notifier.notify_one();
-		if let Some(event_loop) = self.event_loop.upgrade() {
-			event_loop.clients.lock().await.remove(self.index);
-		}
+}
+impl Debug for Client {
+	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+		f.debug_struct("Client")
+			.field("pid", &self.pid)
+			.field("exe", &self.exe)
+			.field("dispatch_join_handle", &self.dispatch_join_handle)
+			.field("flush_join_handle", &self.flush_join_handle)
+			.field("disconnect_status", &self.disconnect_status)
+			.field("base_resource_prefixes", &self.base_resource_prefixes)
+			.field("state", &self.state)
+			.finish()
 	}
 }
 impl Drop for Client {
 	fn drop(&mut self) {
-		self.stop_notifier.notify_one();
-		CLIENTS.remove(self);
-		println!("Client disconnected");
+		info!(
+			pid = self.pid,
+			exe = self
+				.exe
+				.as_ref()
+				.and_then(|exe| exe.to_str().map(|s| s.to_string())),
+			disconnect_status = match self.disconnect_status.take() {
+				Some(Ok(_)) => "Graceful disconnect".to_string(),
+				Some(Err(e)) => format!("Error: {}", e.root_cause()),
+				None => "Unknown".to_string(),
+			},
+			"Client disconnected"
+		);
 	}
 }

src/core/client_state.rs

@@ -0,0 +1,119 @@
+use super::client::{Client, get_env};
+use crate::nodes::{Node, root::ClientState, spatial::Spatial};
+use glam::Mat4;
+use parking_lot::Mutex;
+use rustc_hash::FxHashMap;
+use serde::{Deserialize, Serialize};
+use std::{
+	path::{Path, PathBuf},
+	process::Command,
+	sync::Arc,
+};
+
+lazy_static::lazy_static! {
+	pub static ref CLIENT_STATES: Mutex<FxHashMap<String, Arc<ClientStateParsed>>> = Default::default();
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct LaunchInfo {
+	pub cmdline: Vec<String>,
+	pub cwd: PathBuf,
+	pub env: FxHashMap<String, String>,
+}
+impl LaunchInfo {
+	fn from_client(client: &Client) -> Option<Self> {
+		Some(LaunchInfo {
+			cmdline: client.get_cmdline()?,
+			cwd: client.get_cwd()?,
+			env: get_env(client.pid?).ok()?,
+		})
+	}
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct ClientStateParsed {
+	pub launch_info: Option<LaunchInfo>,
+	pub data: Vec<u8>,
+	pub root: Mat4,
+	pub spatial_anchors: FxHashMap<String, Mat4>,
+}
+impl ClientStateParsed {
+	pub fn from_deserialized(client: &Client, state: ClientState) -> Self {
+		ClientStateParsed {
+			launch_info: LaunchInfo::from_client(client),
+			data: state.data.unwrap_or_default(),
+			root: Self::spatial_transform(client, state.root).unwrap_or_default(),
+			spatial_anchors: state
+				.spatial_anchors
+				.into_iter()
+				.filter_map(|(k, v)| Some((k, Self::spatial_transform(client, v)?)))
+				.collect(),
+		}
+	}
+	fn spatial_transform(client: &Client, id: u64) -> Option<Mat4> {
+		let node = client.scenegraph.get_node(id)?;
+		let spatial = node.get_aspect::<Spatial>().ok()?;
+		Some(spatial.global_transform())
+	}
+
+	pub fn token(self) -> String {
+		let token = nanoid::nanoid!();
+		CLIENT_STATES.lock().insert(token.clone(), Arc::new(self));
+		token
+	}
+	pub fn from_file(file: &Path) -> Option<Self> {
+		let file_string = std::fs::read_to_string(file).ok()?;
+		toml::from_str(&file_string).ok()
+	}
+	pub fn to_file(&self, directory: &Path) {
+		let app_name = self
+			.launch_info
+			.as_ref()
+			.map(|l| l.cmdline.first().unwrap().split('/').next_back().unwrap())
+			.unwrap_or("unknown");
+		let state_file_path = directory
+			.join(format!("{app_name}-{}", nanoid::nanoid!()))
+			.with_extension("toml");
+
+		std::fs::write(state_file_path, toml::to_string(&self).unwrap()).unwrap();
+	}
+
+	pub fn apply_to(&self, client: &Arc<Client>) -> ClientState {
+		if let Some(root) = client.root.get() {
+			root.set_transform(self.root)
+		}
+		ClientState {
+			data: Some(self.data.clone()),
+			root: 0,
+			spatial_anchors: self
+				.spatial_anchors
+				.iter()
+				.map(|(k, v)| {
+					let node = Node::generate(client, true).add_to_scenegraph().unwrap();
+					Spatial::add_to(&node, None, *v, false);
+					(k.clone(), node.get_id())
+				})
+				.collect(),
+		}
+	}
+	pub fn launch_command(self) -> Option<Command> {
+		let launch_info = self.launch_info.as_ref()?;
+		let mut cmdline = launch_info.cmdline.iter();
+		let mut command = Command::new(cmdline.next()?);
+		command.args(cmdline);
+		command.current_dir(&launch_info.cwd);
+		command.envs(launch_info.env.iter());
+		command.env("STARDUST_STARTUP_TOKEN", self.token());
+		Some(command)
+	}
+}
+impl Default for ClientStateParsed {
+	fn default() -> Self {
+		Self {
+			launch_info: None,
+			data: Default::default(),
+			root: Mat4::IDENTITY,
+			spatial_anchors: Default::default(),
+		}
+	}
+}

src/core/color.rs

@@ -0,0 +1,12 @@
+use stardust_xr::values::color::{AlphaColor, Rgb, color_space::LinearRgb};
+
+pub trait ColorConvert {
+	fn to_bevy(&self) -> bevy::color::Color;
+}
+// even tho its supposed to be linear the values have to be interpreted as Srgba to produce the
+// correct result because StereoKit used Srgba while it was assumed that is uses linear rgba
+impl ColorConvert for AlphaColor<f32, Rgb<f32, LinearRgb>> {
+	fn to_bevy(&self) -> bevy::color::Color {
+		bevy::color::Color::srgba(self.c.r, self.c.g, self.c.b, self.a)
+	}
+}

src/core/delta.rs

@@ -0,0 +1,47 @@
+use std::ops::{Deref, DerefMut};
+
+#[derive(Debug)]
+pub struct Delta<T> {
+	value: T,
+	changed: bool,
+}
+#[allow(dead_code)]
+impl<T> Delta<T> {
+	pub const fn new(value: T) -> Self {
+		Delta {
+			value,
+			changed: false,
+		}
+	}
+	pub fn peek_delta(&self) -> Option<&T> {
+		self.changed.then_some(&self.value)
+	}
+	pub fn delta(&mut self) -> Option<&mut T> {
+		let delta = self.changed.then_some(&mut self.value);
+		self.changed = false;
+		delta
+	}
+	pub fn mark_changed(&mut self) {
+		self.changed = true;
+	}
+	pub const fn value(&self) -> &T {
+		&self.value
+	}
+	pub fn value_mut(&mut self) -> &mut T {
+		self.mark_changed();
+		&mut self.value
+	}
+}
+impl<T> Deref for Delta<T> {
+	type Target = T;
+
+	fn deref(&self) -> &Self::Target {
+		&self.value
+	}
+}
+impl<T> DerefMut for Delta<T> {
+	fn deref_mut(&mut self) -> &mut Self::Target {
+		self.mark_changed();
+		&mut self.value
+	}
+}

src/core/destroy_queue.rs

@@ -1,12 +0,0 @@
-use parking_lot::Mutex;
-use std::any::Any;
-
-static MAIN_DESTROY_QUEUE: Mutex<Vec<Box<dyn Any + Send + Sync>>> = Mutex::new(Vec::new());
-
-pub fn add<T: Any + Sync + Send>(thing: T) {
-	MAIN_DESTROY_QUEUE.lock().push(Box::new(thing));
-}
-
-pub fn clear() {
-	MAIN_DESTROY_QUEUE.lock().clear();
-}

src/core/entity_handle.rs

@@ -0,0 +1,33 @@
+use bevy::prelude::*;
+
+use super::bevy_channel::{BevyChannel, BevyChannelReader};
+pub struct EntityHandlePlugin;
+
+impl Plugin for EntityHandlePlugin {
+	fn build(&self, app: &mut App) {
+		DESTROY.init(app);
+		app.add_systems(PreUpdate, despawn);
+	}
+}
+
+fn despawn(mut cmds: Commands, mut reader: ResMut<BevyChannelReader<Entity>>) {
+	while let Some(e) = reader.read() {
+		cmds.entity(e).try_despawn();
+	}
+}
+
+static DESTROY: BevyChannel<Entity> = BevyChannel::new();
+#[derive(Deref, DerefMut, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)]
+pub struct EntityHandle(pub Entity);
+impl Drop for EntityHandle {
+	fn drop(&mut self) {
+		if DESTROY.send(self.0).is_none() {
+			error!("Entity Destroy channel not open");
+		}
+	}
+}
+impl From<Entity> for EntityHandle {
+	fn from(value: Entity) -> Self {
+		Self(value)
+	}
+}

src/core/error.rs

@@ -0,0 +1,75 @@
+use std::any::TypeId;
+
+use color_eyre::eyre::Report;
+use stardust_xr::{
+	messenger::MessengerError,
+	schemas::flex::{
+		FlexSerializeError,
+		flexbuffers::{DeserializationError, ReaderError},
+	},
+};
+use thiserror::Error;
+
+pub type Result<T, E = ServerError> = std::result::Result<T, E>;
+
+#[derive(Error, Debug)]
+pub enum ServerError {
+	#[error("Internal: Unable to get client")]
+	NoClient,
+	#[error("Messenger does not exist for this node")]
+	NoMessenger,
+	#[error("Messenger error: {0}")]
+	MessengerError(#[from] MessengerError),
+	#[error("Remote method error: {0}")]
+	RemoteMethodError(String),
+	#[error("Serialization error: {0}")]
+	SerializationError(#[from] FlexSerializeError),
+	#[error("Deserialization error: {0}")]
+	DeserializationError(#[from] DeserializationError),
+	#[error("Reader error: {0}")]
+	ReaderError(#[from] ReaderError),
+	#[cfg(feature = "wayland")]
+	#[error("Wayland error: {0}")]
+	WaylandError(waynest::server::Error),
+	#[error("Aspect {} does not exist for node", 0.to_string())]
+	NoAspect(TypeId),
+	#[error("{0}")]
+	Report(#[from] Report),
+}
+
+#[macro_export]
+macro_rules! bail {
+    ($msg:literal $(,)?) => {
+        return Err($crate::core::error::ServerError::from(color_eyre::eyre::eyre!($msg)));
+    };
+    ($err:expr $(,)?) => {
+        return Err($crate::core::error::ServerError::from(color_eyre::eyre::eyre!($err)));
+    };
+    ($fmt:expr, $($arg:tt)*) => {
+        return Err($crate::core::error::ServerError::from(color_eyre::eyre::eyre!($fmt, $($arg)*)));
+    };
+}
+
+#[macro_export]
+macro_rules! ensure {
+    ($cond:expr $(,)?) => {
+        if !$cond {
+            $crate::ensure!($cond, concat!("Condition failed: `", stringify!($cond), "`"))
+        }
+    };
+    ($cond:expr, $msg:literal $(,)?) => {
+        if !$cond {
+            return Err($crate::core::error::ServerError::from(color_eyre::eyre::eyre!($msg)));
+        }
+    };
+    ($cond:expr, $err:expr $(,)?) => {
+        if !$cond {
+            return Err($crate::core::error::ServerError::from(color_eyre::eyre::eyre!($err)));
+        }
+    };
+    ($cond:expr, $fmt:expr, $($arg:tt)*) => {
+        if !$cond {
+            return Err($crate::core::error::ServerError::from(color_eyre::eyre::eyre!($fmt, $($arg)*)));
+        }
+    };
+}

src/core/eventloop.rs

@@ -1,61 +0,0 @@
-use super::client::Client;
-use anyhow::Result;
-use slab::Slab;
-use stardust_xr::server;
-use std::sync::atomic::AtomicU64;
-use std::sync::Arc;
-use tokio::net::UnixListener;
-use tokio::sync::{Mutex, Notify};
-use tokio::task::JoinHandle;
-
-pub static FRAME: AtomicU64 = AtomicU64::new(0);
-
-pub struct EventLoop {
-	pub socket_path: String,
-	stop_notifier: Arc<Notify>,
-	pub clients: Mutex<Slab<Arc<Client>>>,
-}
-
-impl EventLoop {
-	pub fn new() -> Result<(Arc<Self>, JoinHandle<Result<()>>)> {
-		let socket_path = server::get_free_socket_path()
-			.ok_or_else(|| std::io::Error::from(std::io::ErrorKind::Other))?;
-		let socket = UnixListener::bind(socket_path.clone())?;
-
-		let event_loop = Arc::new(EventLoop {
-			socket_path,
-			stop_notifier: Default::default(),
-			clients: Mutex::new(Slab::new()),
-		});
-
-		let event_loop_join_handle = tokio::spawn({
-			let event_loop = event_loop.clone();
-			async move { EventLoop::event_loop(socket, event_loop).await }
-		});
-
-		Ok((event_loop, event_loop_join_handle))
-	}
-
-	async fn event_loop(socket: UnixListener, event_loop: Arc<EventLoop>) -> Result<()> {
-		let event_loop_async = async {
-			loop {
-				let (socket, _) = socket.accept().await?;
-				let mut clients = event_loop.clients.lock().await;
-				let vacant_client = clients.vacant_entry();
-				let idx = vacant_client.key();
-				vacant_client.insert(Client::from_connection(idx, &event_loop, socket));
-			}
-		};
-
-		tokio::select! {
-			_ = event_loop.stop_notifier.notified() => Ok(()),
-			e = event_loop_async => e,
-		}
-	}
-}
-
-impl Drop for EventLoop {
-	fn drop(&mut self) {
-		self.stop_notifier.notify_one();
-	}
-}

src/core/mod.rs

@@ -1,7 +1,11 @@
+pub mod bevy_channel;
 pub mod client;
-pub mod destroy_queue;
-pub mod eventloop;
-pub mod nodelist;
+pub mod client_state;
+pub mod color;
+pub mod delta;
+pub mod entity_handle;
+pub mod error;
 pub mod registry;
 pub mod resource;
 pub mod scenegraph;
+pub mod task;

src/core/nodelist.rs

@@ -1,29 +0,0 @@
-use crate::nodes::Node;
-use parking_lot::Mutex;
-use std::sync::Weak;
-
-#[derive(Default)]
-pub struct LifeLinkedNodeList {
-	nodes: Mutex<Vec<Weak<Node>>>,
-}
-impl LifeLinkedNodeList {
-	pub fn add(&self, node: Weak<Node>) {
-		self.nodes.lock().push(node);
-	}
-
-	pub fn clear(&self) {
-		self.nodes
-			.lock()
-			.iter()
-			.filter_map(|node| node.upgrade())
-			.for_each(|node| {
-				node.destroy();
-			});
-		self.nodes.lock().clear();
-	}
-}
-impl Drop for LifeLinkedNodeList {
-	fn drop(&mut self) {
-		self.clear();
-	}
-}

src/core/registry.rs

@@ -1,17 +1,18 @@
 #![allow(dead_code)]
 
-use std::ptr;
-use std::sync::{Arc, Weak};
-
-use once_cell::sync::Lazy;
-use parking_lot::Mutex;
+use dashmap::DashMap;
+use parking_lot::{MappedMutexGuard, Mutex, MutexGuard, const_mutex};
 use rustc_hash::FxHashMap;
+use std::ops::Deref;
+use std::ptr;
+use std::sync::{Arc, LazyLock, Weak};
 
-pub struct Registry<T: Send + Sync + ?Sized>(Lazy<Mutex<FxHashMap<usize, Weak<T>>>>);
+#[derive(Debug)]
+pub struct Registry<T: Send + Sync + ?Sized>(MaybeLazy<DashMap<usize, Weak<T>>>);
 
 impl<T: Send + Sync + ?Sized> Registry<T> {
 	pub const fn new() -> Self {
-		Registry(Lazy::new(|| Mutex::new(FxHashMap::default())))
+		Registry(MaybeLazy::Lazy(LazyLock::new(DashMap::default)))
 	}
 	pub fn add(&self, t: T) -> Arc<T>
 	where
@@ -23,22 +24,159 @@ impl<T: Send + Sync + ?Sized> Registry<T> {
 	}
 	pub fn add_raw(&self, t: &Arc<T>) {
 		self.0
-			.lock()
 			.insert(Arc::as_ptr(t) as *const () as usize, Arc::downgrade(t));
 	}
+	pub fn contains(&self, t: &T) -> bool {
+		self.0
+			.contains_key(&(ptr::addr_of!(*t) as *const () as usize))
+	}
+	pub fn get_changes(old: &Registry<T>, new: &Registry<T>) -> (Vec<Arc<T>>, Vec<Arc<T>>) {
+		let mut added = Vec::new();
+		let mut removed = Vec::new();
+
+		for pair in new.0.iter() {
+			let (id, entry) = pair.pair();
+			if let Some(entry) = entry.upgrade() {
+				if !old.0.contains_key(id) {
+					added.push(entry);
+				}
+			}
+		}
+		for pair in old.0.iter() {
+			let (id, entry) = pair.pair();
+			if let Some(entry) = entry.upgrade() {
+				if !new.0.contains_key(id) {
+					removed.push(entry);
+				}
+			}
+		}
+		(added, removed)
+	}
 	pub fn get_valid_contents(&self) -> Vec<Arc<T>> {
 		self.0
-			.lock()
 			.iter()
-			.filter_map(|pair| pair.1.upgrade())
+			.filter_map(|pair| pair.value().upgrade())
 			.collect()
 	}
+	pub fn set(&self, other: &Registry<T>) {
+		self.clear();
+		for (key, value) in other.0.deref().clone().into_iter() {
+			self.0.insert(key, value);
+		}
+	}
+	pub fn take_valid_contents(&self) -> Vec<Arc<T>> {
+		let contents = self.get_valid_contents();
+		self.0.clear();
+		contents
+	}
+	pub fn retain<F: Fn(&Arc<T>) -> bool>(&self, f: F) {
+		self.0.retain(|_, v| {
+			let Some(v) = v.upgrade() else {
+				// why would we want to retain things we can't upgrade?
+				return true;
+			};
+			(f)(&v)
+		})
+	}
 	pub fn remove(&self, t: &T) {
-		self.0
-			.lock()
-			.remove(&(ptr::addr_of!(*t) as *const () as usize));
+		self.0.remove(&(ptr::addr_of!(*t) as *const () as usize));
 	}
 	pub fn clear(&self) {
-		self.0.lock().clear();
+		self.0.clear();
+	}
+	pub fn is_empty(&self) -> bool {
+		if self.0.is_empty() {
+			return true;
+		}
+		self.0.iter().all(|v| v.value().strong_count() == 0)
+	}
+}
+
+impl<T: Send + Sync + ?Sized> Clone for Registry<T> {
+	fn clone(&self) -> Self {
+		Self(self.0.clone())
+	}
+}
+impl<T: Send + Sync + ?Sized> Default for Registry<T> {
+	fn default() -> Self {
+		Self::new()
+	}
+}
+
+impl<T: Send + Sync + Sized> FromIterator<Arc<T>> for Registry<T> {
+	fn from_iter<I: IntoIterator<Item = Arc<T>>>(iter: I) -> Self {
+		Registry(MaybeLazy::NonLazy(
+			iter.into_iter()
+				.map(|i| (Arc::as_ptr(&i) as usize, Arc::downgrade(&i)))
+				.collect(),
+		))
+	}
+}
+
+#[derive(Debug)]
+enum MaybeLazy<T> {
+	Lazy(LazyLock<T>),
+	NonLazy(T),
+}
+impl<T: Clone> Clone for MaybeLazy<T> {
+	fn clone(&self) -> Self {
+		match self {
+			MaybeLazy::Lazy(lazy_lock) => Self::NonLazy(lazy_lock.deref().clone()),
+			MaybeLazy::NonLazy(v) => Self::NonLazy(v.clone()),
+		}
+	}
+}
+impl<T> Deref for MaybeLazy<T> {
+	type Target = T;
+
+	fn deref(&self) -> &Self::Target {
+		match self {
+			MaybeLazy::Lazy(lazy_lock) => lazy_lock,
+			MaybeLazy::NonLazy(v) => v,
+		}
+	}
+}
+
+pub struct OwnedRegistry<T: Send + Sync + ?Sized>(Mutex<Option<FxHashMap<usize, Arc<T>>>>);
+
+impl<T: Send + Sync + ?Sized> OwnedRegistry<T> {
+	pub const fn new() -> Self {
+		OwnedRegistry(const_mutex(None))
+	}
+	fn lock(&self) -> MappedMutexGuard<FxHashMap<usize, Arc<T>>> {
+		MutexGuard::map(self.0.lock(), |r| r.get_or_insert_with(FxHashMap::default))
+	}
+	pub fn add(&self, t: T) -> Arc<T>
+	where
+		T: Sized,
+	{
+		let t_arc = Arc::new(t);
+		self.add_raw(t_arc.clone());
+		t_arc
+	}
+	pub fn add_raw(&self, t: Arc<T>) {
+		self.lock().insert(Arc::as_ptr(&t) as *const () as usize, t);
+	}
+	pub fn get_vec(&self) -> Vec<Arc<T>> {
+		self.lock().values().cloned().collect::<Vec<_>>()
+	}
+	pub fn contains(&self, t: &T) -> bool {
+		self.lock()
+			.contains_key(&(ptr::addr_of!(*t) as *const () as usize))
+	}
+	pub fn remove(&self, t: &T) -> Option<Arc<T>>
+	where
+		T: Sized,
+	{
+		self.lock()
+			.remove(&(ptr::addr_of!(*t) as *const () as usize))
+	}
+	pub fn clear(&self) {
+		self.lock().clear();
+	}
+}
+impl<T: Send + Sync + ?Sized> Clone for OwnedRegistry<T> {
+	fn clone(&self) -> Self {
+		Self(Mutex::new(self.0.lock().clone()))
 	}
 }

src/core/resource.rs

@@ -1,71 +1,49 @@
-use anyhow::anyhow;
-use serde::{de::Visitor, Deserialize};
-use std::path::PathBuf;
+use stardust_xr::values::ResourceID;
+use std::{
+	ffi::OsStr,
+	path::{Path, PathBuf},
+};
 
-pub enum ResourceID {
-	File(PathBuf),
-	Namespaced { namespace: String, path: PathBuf },
+use super::client::Client;
+
+lazy_static::lazy_static! {
+	static ref THEMES: Vec<PathBuf> = std::env::var("STARDUST_THEMES").map(|s| s.split(':').map(PathBuf::from).collect()).unwrap_or_default();
 }
-impl ResourceID {
-	pub fn get_file(&self, prefixes: &[PathBuf]) -> Option<PathBuf> {
-		match self {
-			ResourceID::File(file) => (file.is_absolute() && file.exists()).then_some(file.clone()),
-			ResourceID::Namespaced { namespace, path } => {
-				for prefix in prefixes {
-					let mut test_path = prefix.clone();
-					test_path.push(namespace.clone());
-					test_path.push(path.clone());
 
-					if test_path.as_path().exists() {
-						return Some(test_path);
-					}
-				}
-				None
-			}
+fn has_extension(path: &Path, extensions: &[&OsStr]) -> bool {
+	if let Some(path_extension) = path.extension() {
+		extensions.contains(&path_extension)
+	} else {
+		false
+	}
+}
+
+pub fn get_resource_file(
+	resource: &ResourceID,
+	client: &Client,
+	extensions: &[&OsStr],
+) -> Option<PathBuf> {
+	match resource {
+		ResourceID::Direct(file) => {
+			(file.is_absolute() && file.exists() && has_extension(file, extensions))
+				.then_some(file.clone())
+		}
+		ResourceID::Namespaced { namespace, path } => {
+			let file_name = path.file_name()?;
+			let base_prefixes = client.base_resource_prefixes.lock().clone();
+			THEMES
+				.iter()
+				.chain(base_prefixes.iter())
+				.filter_map(|prefix| {
+					let prefixed_path = prefix.clone().join(namespace).join(path);
+					let parent = prefixed_path.parent()?;
+					std::fs::read_dir(parent).ok()
+				})
+				.flatten()
+				.filter_map(|item| item.ok())
+				.map(|dir_entry| dir_entry.path())
+				.filter(|path| path.file_stem() == Some(file_name))
+				.find(|path| has_extension(path, extensions))
 		}
 	}
 }
-impl<'de> Deserialize<'de> for ResourceID {
-	fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
-	where
-		D: serde::Deserializer<'de>,
-	{
-		deserializer.deserialize_any(ResourceVisitor)
-	}
-}
-
-struct ResourceVisitor;
-impl<'de> Visitor<'de> for ResourceVisitor {
-	type Value = ResourceID;
-
-	fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
-		formatter.write_str("A string containing an absolute path to file or \"[namespace]:[path]\" for a namespaced resource")
-	}
-
-	fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
-	where
-		E: serde::de::Error,
-	{
-		Ok(if v.starts_with('/') {
-			let path = PathBuf::from(v);
-			path.metadata().map_err(serde::de::Error::custom)?;
-			ResourceID::File(path)
-		} else if let Some((namespace, path)) = v.split_once(':') {
-			ResourceID::Namespaced {
-				namespace: namespace.to_string(),
-				path: PathBuf::from(path),
-			}
-		} else {
-			return Err(serde::de::Error::custom(anyhow!(
-				"Invalid format for string"
-			)));
-		})
-	}
-
-	fn visit_string<E>(self, v: String) -> Result<Self::Value, E>
-	where
-		E: serde::de::Error,
-	{
-		self.visit_str(&v)
-	}
-}

src/core/scenegraph.rs

@@ -1,24 +1,28 @@
-use crate::core::client::Client;
+use crate::core::error::Result;
 use crate::nodes::Node;
-use anyhow::Result;
-use once_cell::sync::OnceCell;
+use crate::nodes::alias::get_original;
+use crate::{core::client::Client, nodes::Message};
+use parking_lot::Mutex;
+use rustc_hash::FxHashMap;
+use serde::Serialize;
 use stardust_xr::scenegraph;
 use stardust_xr::scenegraph::ScenegraphError;
-use std::sync::{Arc, Weak};
-
-use core::hash::BuildHasherDefault;
-use dashmap::DashMap;
-use rustc_hash::FxHasher;
+use stardust_xr::schemas::flex::serialize;
+use std::future::Future;
+use std::os::fd::OwnedFd;
+use std::sync::{Arc, OnceLock, Weak};
+use tokio::sync::oneshot;
+use tracing::{debug, debug_span};
 
 #[derive(Default)]
 pub struct Scenegraph {
-	pub(super) client: OnceCell<Weak<Client>>,
-	nodes: DashMap<String, Arc<Node>, BuildHasherDefault<FxHasher>>,
+	pub(super) client: OnceLock<Weak<Client>>,
+	nodes: Mutex<FxHashMap<u64, Arc<Node>>>,
 }
 
 impl Scenegraph {
-	pub fn get_client(&self) -> Arc<Client> {
-		self.client.get().unwrap().upgrade().unwrap()
+	pub fn get_client(&self) -> Option<Arc<Client>> {
+		self.client.get()?.upgrade()
 	}
 
 	pub fn add_node(&self, node: Node) -> Arc<Node> {
@@ -27,38 +31,110 @@ impl Scenegraph {
 		node_arc
 	}
 	pub fn add_node_raw(&self, node: Arc<Node>) {
-		let path = node.get_path().to_string();
-		self.nodes.insert(path, node);
+		debug!(node = ?&*node, "Add node");
+		self.nodes.lock().insert(node.get_id(), node);
 	}
 
-	pub fn get_node(&self, path: &str) -> Option<Arc<Node>> {
-		let mut node = self.nodes.get(path)?.clone();
-		if let Some(alias) = node.alias.get() {
-			node = alias.original.upgrade()?;
-		}
-		Some(node)
+	pub fn get_node(&self, node: u64) -> Option<Arc<Node>> {
+		let node = self.nodes.lock().get(&node)?.clone();
+		get_original(node, true)
 	}
 
-	pub fn remove_node(&self, path: &str) -> Option<Arc<Node>> {
-		let (_, node) = self.nodes.remove(path)?;
-		Some(node)
+	pub fn remove_node(&self, node: u64) -> Option<Arc<Node>> {
+		debug!(node, "Remove node");
+		self.nodes.lock().remove(&node)
 	}
 }
 
+pub type MethodResponse = Result<(Vec<u8>, Vec<OwnedFd>), ScenegraphError>;
+pub struct MethodResponseSender(oneshot::Sender<MethodResponse>);
+impl MethodResponseSender {
+	pub fn send(self, t: Result<Message, ScenegraphError>) {
+		let _ = self.0.send(t.map(|m| (m.data, m.fds)));
+	}
+	// pub fn send_method_return<T: Serialize>(
+	// 	self,
+	// 	result: color_eyre::eyre::Result<(T, Vec<OwnedFd>)>,
+	// ) {
+	// 	let _ = self.0.send(map_method_return(result));
+	// }
+	pub fn wrap_sync<F: FnOnce() -> crate::core::error::Result<Message>>(self, f: F) {
+		self.send(f().map_err(|e| ScenegraphError::MemberError {
+			error: e.to_string(),
+		}))
+	}
+	pub fn wrap_async<T: Serialize>(
+		self,
+		f: impl Future<Output = Result<(T, Vec<OwnedFd>)>> + Send + 'static,
+	) {
+		tokio::task::spawn(async move { self.0.send(map_method_return(f.await)) });
+	}
+}
+fn map_method_return<T: Serialize>(
+	result: Result<(T, Vec<OwnedFd>)>,
+) -> Result<(Vec<u8>, Vec<OwnedFd>), ScenegraphError> {
+	let (value, fds) = result.map_err(|e| ScenegraphError::MemberError {
+		error: e.to_string(),
+	})?;
+
+	let serialized_value = serialize(value).map_err(|e| ScenegraphError::MemberError {
+		error: format!("Internal: Serialization failed: {e}"),
+	})?;
+	Ok((serialized_value, fds))
+}
 impl scenegraph::Scenegraph for Scenegraph {
-	fn send_signal(&self, path: &str, method: &str, data: &[u8]) -> Result<(), ScenegraphError> {
-		self.get_node(path)
-			.ok_or(ScenegraphError::NodeNotFound)?
-			.send_local_signal(self.get_client(), method, data)
+	fn send_signal(
+		&self,
+		node_id: u64,
+		aspect_id: u64,
+		method: u64,
+		data: &[u8],
+		fds: Vec<OwnedFd>,
+	) -> Result<(), ScenegraphError> {
+		let Some(client) = self.get_client() else {
+			return Err(ScenegraphError::NodeNotFound);
+		};
+		debug_span!("Handle signal", aspect_id, node_id, method).in_scope(|| {
+			self.get_node(node_id)
+				.ok_or(ScenegraphError::NodeNotFound)?
+				.send_local_signal(
+					client,
+					aspect_id,
+					method,
+					Message {
+						data: data.to_vec(),
+						fds,
+					},
+				)
+		})
 	}
 	fn execute_method(
 		&self,
-		path: &str,
-		method: &str,
+		node_id: u64,
+		aspect_id: u64,
+		method: u64,
 		data: &[u8],
-	) -> Result<Vec<u8>, ScenegraphError> {
-		self.get_node(path)
-			.ok_or(ScenegraphError::NodeNotFound)?
-			.execute_local_method(self.get_client(), method, data)
+		fds: Vec<OwnedFd>,
+		response: oneshot::Sender<Result<(Vec<u8>, Vec<OwnedFd>), ScenegraphError>>,
+	) {
+		let Some(client) = self.get_client() else {
+			let _ = response.send(Err(ScenegraphError::NodeNotFound));
+			return;
+		};
+		debug!(aspect_id, node_id, method, "Handle method");
+		let Some(node) = self.get_node(node_id) else {
+			let _ = response.send(Err(ScenegraphError::NodeNotFound));
+			return;
+		};
+		node.execute_local_method(
+			client,
+			aspect_id,
+			method,
+			Message {
+				data: data.to_vec(),
+				fds,
+			},
+			MethodResponseSender(response),
+		);
 	}
 }

src/core/task.rs

@@ -0,0 +1,22 @@
+use color_eyre::eyre::Result;
+use std::future::Future;
+use tokio::task::JoinHandle;
+
+#[allow(unused_variables)]
+pub fn new<
+	F: FnOnce() -> S,
+	S: AsRef<str>,
+	A: Future<Output = O> + Send + 'static,
+	O: Send + 'static,
+>(
+	name_fn: F,
+	async_future: A,
+) -> Result<JoinHandle<O>> {
+	#[cfg(not(feature = "profile_tokio"))]
+	let result = Ok(tokio::task::spawn(async_future));
+	#[cfg(feature = "profile_tokio")]
+	let result = tokio::task::Builder::new()
+		.name(name_fn().as_ref())
+		.spawn(async_future);
+	result
+}

src/main.rs

@@ -1,172 +1,498 @@
+#![allow(clippy::empty_docs)]
+#![allow(clippy::too_many_arguments)]
+#![allow(clippy::type_complexity)]
 mod core;
 mod nodes;
 mod objects;
+mod session;
+pub mod tracking_offset;
+#[cfg(feature = "wayland")]
 mod wayland;
 
-use crate::core::destroy_queue;
-use crate::nodes::{drawable, hmd, input};
-use crate::objects::input::mouse_pointer::MousePointer;
-use crate::objects::input::sk_controller::SkController;
-use crate::objects::input::sk_hand::SkHand;
-use crate::wayland::Wayland;
+use crate::nodes::input;
 
-use self::core::eventloop::EventLoop;
-use anyhow::Result;
+use bevy::MinimalPlugins;
+use bevy::a11y::AccessibilityPlugin;
+use bevy::app::{App, ScheduleRunnerPlugin, TerminalCtrlCHandlerPlugin};
+use bevy::asset::{AssetMetaCheck, UnapprovedPathMode};
+use bevy::audio::AudioPlugin;
+use bevy::core_pipeline::CorePipelinePlugin;
+use bevy::diagnostic::DiagnosticsPlugin;
+use bevy::ecs::schedule::{ExecutorKind, ScheduleLabel};
+use bevy::gizmos::GizmoPlugin;
+use bevy::gltf::GltfPlugin;
+use bevy::input::InputPlugin;
+use bevy::pbr::PbrPlugin;
+use bevy::render::settings::{Backends, RenderCreation, WgpuSettings};
+use bevy::render::{RenderDebugFlags, RenderPlugin};
+use bevy::scene::ScenePlugin;
+use bevy::window::{CompositeAlphaMode, PresentMode};
+use bevy::winit::{WakeUp, WinitPlugin, WinitSettings, UpdateMode};
+use bevy_dmabuf::import::DmabufImportPlugin;
+use bevy_mod_openxr::action_set_attaching::OxrActionAttachingPlugin;
+use bevy_mod_openxr::action_set_syncing::OxrActionSyncingPlugin;
+use bevy_mod_openxr::add_xr_plugins;
+use bevy_mod_openxr::exts::OxrExtensions;
+use bevy_mod_openxr::features::overlay::OxrOverlaySettings;
+use bevy_mod_openxr::graphics::{GraphicsBackend, OxrManualGraphicsConfig};
+use bevy_mod_openxr::init::{OxrInitPlugin, should_run_frame_loop};
+use bevy_mod_openxr::reference_space::OxrReferenceSpacePlugin;
+use bevy_mod_openxr::render::{OxrRenderPlugin, OxrWaitFrameSystem};
+use bevy_mod_openxr::resources::{OxrFrameState, OxrFrameWaiter, OxrSessionConfig};
+use bevy_mod_openxr::types::AppInfo;
+use bevy_mod_xr::camera::XrProjection;
+use bevy_mod_xr::session::{XrFirst, XrHandleEvents, XrSessionPlugin};
 use clap::Parser;
+use core::client::{Client, tick_internal_client};
+use core::entity_handle::EntityHandlePlugin;
+use core::task;
 use directories::ProjectDirs;
-use slog::Drain;
-use std::sync::Arc;
-use stereokit::input::Handed;
-use stereokit::lifecycle::DepthMode;
-use stereokit::render::SphericalHarmonics;
-use stereokit::texture::Texture;
-use stereokit::{lifecycle::DisplayMode, Settings};
-use tokio::{runtime::Handle, sync::oneshot};
+use nodes::audio::AudioNodePlugin;
+use nodes::drawable::lines::LinesNodePlugin;
+use nodes::drawable::model::ModelNodePlugin;
+use nodes::drawable::text::TextNodePlugin;
+use nodes::fields::FieldDebugGizmoPlugin;
+use nodes::spatial::SpatialNodePlugin;
+use objects::hmd::HmdPlugin;
+use objects::input::mouse_pointer::FlatscreenInputPlugin;
+use objects::input::oxr_controller::ControllerPlugin;
+use objects::input::oxr_hand::HandPlugin;
+use objects::play_space::PlaySpacePlugin;
+use openxr::{EnvironmentBlendMode, ReferenceSpaceType};
+use session::{launch_start, save_session};
+use stardust_xr::schemas::dbus::object_registry::ObjectRegistry;
+use stardust_xr::server;
+use std::ops::DerefMut as _;
+use std::path::PathBuf;
+use std::sync::{Arc, OnceLock};
+use tokio::net::UnixListener;
+use tokio::sync::Notify;
+use tokio::task::JoinError;
+use tracing::metadata::LevelFilter;
+use tracing::{error, info};
+use tracing_subscriber::{EnvFilter, fmt, prelude::*};
+use tracking_offset::TrackingOffsetPlugin;
+use wayland::{Wayland, WaylandPlugin};
+use zbus::Connection;
+use zbus::fdo::ObjectManager;
 
-#[derive(Parser)]
+use bevy::prelude::*;
+
+#[derive(Debug, Clone, Parser)]
 #[clap(author, version, about, long_about = None)]
 struct CliArgs {
 	/// Force flatscreen mode and use the mouse pointer as a 3D pointer
 	#[clap(short, long, action)]
 	flatscreen: bool,
 
+	/// Creates a transparent window fot the flatscreen mode
+	#[clap(short, long, action)]
+	transparent_flatscreen: bool,
+
+	/// If monado insists on emulating them, set this flag...we want the raw input
+	#[clap(long)]
+	disable_controllers: bool,
+	/// If monado insists on emulating , set this flag...we want the raw input
+	#[clap(long)]
+	disable_hands: bool,
+
 	/// Run Stardust XR as an overlay with given priority
 	#[clap(id = "PRIORITY", short = 'o', long = "overlay", action)]
 	overlay_priority: Option<u32>,
+
+	/// Debug the clients started by the server
+	#[clap(short = 'd', long = "debug", action)]
+	debug_launched_clients: bool,
+
+	/// Run a script when ready for clients to connect. If this is not set the script at $HOME/.config/stardust/startup will be ran if it exists.
+	#[clap(id = "PATH", short = 'e', long = "execute-startup-script", action)]
+	startup_script: Option<PathBuf>,
+
+	/// Restore the session with the given ID (or `latest`), ignoring the startup script. Sessions are stored in directories at `~/.local/state/stardust/`.
+	#[clap(id = "SESSION_ID", long = "restore", action)]
+	restore: Option<String>,
 }
 
-fn main() -> Result<()> {
-	let project_dirs = ProjectDirs::from("", "", "stardust").unwrap();
-	let cli_args = Arc::new(CliArgs::parse());
-	let log = ::slog::Logger::root(::slog_stdlog::StdLog.fuse(), slog::o!());
-	slog_stdlog::init()?;
+pub type BevyMaterial = StandardMaterial;
 
-	let mut stereokit = Settings::default()
-		.app_name("Stardust XR")
-		.overlay_app(cli_args.overlay_priority.is_some())
-		.overlay_priority(cli_args.overlay_priority.unwrap_or(u32::MAX))
-		.disable_desktop_input_window(true)
-		.display_preference(if cli_args.flatscreen {
-			DisplayMode::Flatscreen
-		} else {
-			DisplayMode::MixedReality
-		})
-		.depth_mode(DepthMode::D32)
-		.init()
-		.expect("StereoKit failed to initialize");
-	println!("Init StereoKit");
+static STARDUST_INSTANCE: OnceLock<String> = OnceLock::new();
 
-	// Skytex/light stuff
-	{
-		let skytex_path = project_dirs.config_dir().join("skytex.hdr");
-		if let Some((tex, light)) = skytex_path
-			.exists()
-			.then(|| Texture::from_cubemap_equirectangular(&stereokit, &skytex_path, true, 100))
-			.flatten()
-		{
-			stereokit.set_skytex(&tex);
-			stereokit.set_skylight(&light);
-		} else if let Some(tex) = Texture::cubemap_from_spherical_harmonics(
-			&stereokit,
-			&SphericalHarmonics::default(),
-			16,
-			0.0,
-			0.0,
-		) {
-			stereokit.set_skytex(&tex);
-		}
-	}
+// #[tokio::main(flavor = "current_thread")]
+#[tokio::main]
+async fn main() -> Result<AppExit, JoinError> {
+	color_eyre::install().unwrap();
 
-	let mouse_pointer = cli_args.flatscreen.then(MousePointer::new);
-	let mut hands =
-		(!cli_args.flatscreen).then(|| [SkHand::new(Handed::Left), SkHand::new(Handed::Right)]);
-	let mut controllers = (!cli_args.flatscreen).then(|| {
-		[
-			SkController::new(Handed::Left),
-			SkController::new(Handed::Right),
-		]
-	});
+	let registry = tracing_subscriber::registry();
 
-	if hands.is_none() {
-		unsafe {
-			stereokit::sys::input_hand_visible(stereokit::sys::handed__handed_left, false as i32);
-			stereokit::sys::input_hand_visible(stereokit::sys::handed__handed_right, false as i32);
-		}
-	}
-
-	let (event_stop_tx, event_stop_rx) = oneshot::channel::<()>();
-	let (handle_sender, handle_receiver) = oneshot::channel::<Handle>();
-	let event_thread = std::thread::Builder::new()
-		.name("event_loop".to_owned())
-		.spawn(move || event_loop(handle_sender, event_stop_rx))?;
-	let _tokio_handle = handle_receiver.blocking_recv()?.enter();
-
-	let mut wayland = Wayland::new(log)?;
-	println!("Stardust ready!");
-	stereokit.run(
-		|sk, draw_ctx| {
-			hmd::frame(sk);
-			wayland.frame(sk);
-			destroy_queue::clear();
-
-			if let Some(mouse_pointer) = &mouse_pointer {
-				mouse_pointer.update(sk);
-			}
-			if let Some(hands) = &mut hands {
-				hands[0].update(sk);
-				hands[1].update(sk);
-			}
-			if let Some(controllers) = &mut controllers {
-				controllers[0].update(sk);
-				controllers[1].update(sk);
-			}
-			input::process_input();
-			nodes::root::Root::logic_step(sk.time_elapsed());
-			drawable::draw(sk, draw_ctx);
-
-			wayland.make_context_current();
-		},
-		|_| {
-			println!("Cleanly shut down StereoKit");
-		},
+	#[cfg(feature = "profile_app")]
+	let registry = registry.with(
+		tracing_tracy::TracyLayer::new(tracing_tracy::DefaultConfig::default())
+			.with_filter(LevelFilter::DEBUG),
 	);
 
-	drop(wayland);
+	#[cfg(feature = "profile_tokio")]
+	let (console_layer, _) = console_subscriber::ConsoleLayer::builder().build();
+	#[cfg(feature = "profile_tokio")]
+	let registry = registry.with(console_layer);
 
-	let _ = event_stop_tx.send(());
-	event_thread
-		.join()
-		.expect("Failed to cleanly shut down event loop")?;
-	println!("Cleanly shut down Stardust");
-	Ok(())
-}
+	let log_layer = fmt::Layer::new()
+		.with_thread_names(true)
+		.with_ansi(true)
+		.with_line_number(true)
+		.with_filter(
+			EnvFilter::builder()
+				.with_default_directive(LevelFilter::WARN.into())
+				.from_env_lossy(),
+		);
+	registry.with(log_layer).init();
 
-// #[tokio::main]
-#[tokio::main(flavor = "current_thread")]
-async fn event_loop(
-	handle_sender: oneshot::Sender<Handle>,
-	stop_rx: oneshot::Receiver<()>,
-) -> anyhow::Result<()> {
-	let _ = handle_sender.send(Handle::current());
-	// console_subscriber::init();
+	let cli_args = CliArgs::parse();
 
-	let (event_loop, event_loop_join_handle) =
-		EventLoop::new().expect("Couldn't create server socket");
-	println!("Init event loop");
-	println!("Stardust socket created at {}", event_loop.socket_path);
+	let socket_path =
+		server::get_free_socket_path().expect("Unable to find a free stardust socket path");
+	STARDUST_INSTANCE.set(socket_path.file_name().unwrap().to_string_lossy().into_owned()).expect("Someone hasn't done their job, yell at Nova because how is this set multiple times what the hell");
+	info!(
+		socket_path = ?socket_path.display(),
+		"Stardust socket created"
+	);
+	let socket =
+		UnixListener::bind(socket_path).expect("Couldn't spawn stardust server at {socket_path}");
+	task::new(|| "client join loop", async move {
+		loop {
+			let Ok((stream, _)) = socket.accept().await else {
+				continue;
+			};
+			if let Err(e) = Client::from_connection(stream) {
+				error!(?e, "Unable to create client from connection");
+			}
+		}
+	})
+	.unwrap();
+	info!("Init client join loop");
 
-	let result = tokio::select! {
-		biased;
-		_ = tokio::signal::ctrl_c() => Ok(()),
-		_ = stop_rx => Ok(()),
-		e = event_loop_join_handle => e?,
-	};
-
-	println!("Cleanly shut down event loop");
-
-	unsafe {
-		stereokit::sys::sk_quit();
+	let project_dirs = ProjectDirs::from("", "", "stardust");
+	if project_dirs.is_none() {
+		error!(
+			"Unable to get Stardust project directories, default skybox and startup script will not work."
+		);
 	}
 
-	result
+	let dbus_connection = Connection::session()
+		.await
+		.expect("Could not open dbus session");
+	// why is this requested here? should there be a specific server bus name that we check
+	// instead?
+	dbus_connection
+		.request_name("org.stardustxr.HMD")
+		.await
+		.expect(
+			"Another instance of the server is running. This is not supported currently (but is planned).",
+		);
+
+	dbus_connection
+		.object_server()
+		.at("/", ObjectManager)
+		.await
+		.expect("Couldn't add the object manager");
+
+	let object_registry = ObjectRegistry::new(&dbus_connection).await.expect(
+		"Couldn't make the object registry to find all objects with given interfaces in d-bus",
+	);
+
+	let _wayland = Wayland::new().expect("Couldn't create Wayland instance");
+
+	let ready_notifier = Arc::new(Notify::new());
+	let io_loop = tokio::task::spawn_blocking({
+		let ready_notifier = ready_notifier.clone();
+		let project_dirs = project_dirs.clone();
+		let cli_args = cli_args.clone();
+		let dbus_connection = dbus_connection.clone();
+		move || {
+			bevy_loop(
+				ready_notifier,
+				project_dirs,
+				cli_args,
+				dbus_connection,
+				object_registry,
+			)
+		}
+	});
+	ready_notifier.notified().await;
+	let mut startup_children = project_dirs
+		.as_ref()
+		.map(|project_dirs| launch_start(&cli_args, project_dirs))
+		.unwrap_or_default();
+	let return_value = io_loop.await;
+	info!("Stopping...");
+	if let Some(project_dirs) = project_dirs {
+		save_session(&project_dirs).await;
+	}
+	for mut startup_child in startup_children.drain(..) {
+		let _ = startup_child.kill();
+	}
+
+	info!("Cleanly shut down Stardust");
+	return_value
+}
+
+// static DEFAULT_SKYTEX: OnceLock<Tex> = OnceLock::new();
+// static DEFAULT_SKYLIGHT: OnceLock<SphericalHarmonics> = OnceLock::new();
+
+#[derive(ScheduleLabel, Hash, Debug, PartialEq, Eq, Clone, Copy)]
+pub struct PreFrameWait;
+#[derive(Resource, Deref)]
+pub struct ObjectRegistryRes(ObjectRegistry);
+#[derive(Resource, Deref)]
+pub struct DbusConnection(Connection);
+
+fn bevy_loop(
+	ready_notifier: Arc<Notify>,
+	_project_dirs: Option<ProjectDirs>,
+	args: CliArgs,
+	dbus_connection: Connection,
+	object_registry: ObjectRegistry,
+) -> AppExit {
+	let mut app = App::new();
+	app.insert_resource(DbusConnection(dbus_connection));
+	app.insert_resource(OxrManualGraphicsConfig {
+		fallback_backend: GraphicsBackend::Vulkan(()),
+		vk_instance_exts: Vec::new(),
+		vk_device_exts: bevy_dmabuf::required_device_extensions(),
+	});
+	app.add_plugins(AssetPlugin {
+		meta_check: AssetMetaCheck::Never,
+		unapproved_path_mode: UnapprovedPathMode::Allow,
+		..default()
+	});
+	let mut plugins = MinimalPlugins
+		.build()
+		.add(DiagnosticsPlugin)
+		.add(TransformPlugin)
+		.add(InputPlugin)
+		.add(AccessibilityPlugin);
+	plugins = plugins
+		.add(TerminalCtrlCHandlerPlugin)
+		// bevy_mod_openxr will replace this, TODO: figure out how to mix this with
+		// bevy-dmabuf
+		.add(RenderPlugin {
+			render_creation: RenderCreation::Automatic(WgpuSettings {
+				backends: Some(Backends::VULKAN),
+				..Default::default()
+			}),
+			..Default::default()
+		})
+		.add(ImagePlugin::default())
+		.add(CorePipelinePlugin)
+		// theoretically we shouldn't need this because of bevy_sk, but everything is tangled in
+		// there and idk what we actually need to run
+		.add(PbrPlugin {
+			// this seems to only apply to StandardMaterial, we don't use that
+			prepass_enabled: true,
+			add_default_deferred_lighting_plugin: false,
+			use_gpu_instance_buffer_builder: true,
+			debug_flags: RenderDebugFlags::default(),
+		})
+		// required for gltf
+		.add(ScenePlugin)
+		.add(GltfPlugin::default())
+		// .add(AnimationPlugin)
+		.add(AudioPlugin::default())
+		.add(GizmoPlugin)
+		.add(WindowPlugin::default())
+		.add(DmabufImportPlugin);
+	let mut task_pool_plugin = TaskPoolPlugin::default();
+	// make tokio work
+	let handle = tokio::runtime::Handle::current();
+	let enter_runtime_context = Arc::new(move || {
+		// TODO: this might be a memory leak
+		std::mem::forget(handle.enter());
+	});
+	task_pool_plugin.task_pool_options.io.on_thread_spawn = Some(enter_runtime_context.clone());
+	task_pool_plugin.task_pool_options.compute.on_thread_spawn =
+		Some(enter_runtime_context.clone());
+	task_pool_plugin
+		.task_pool_options
+		.async_compute
+		.on_thread_spawn = Some(enter_runtime_context.clone());
+	plugins = plugins.set(task_pool_plugin);
+	let flatscreenmode = args.flatscreen
+		|| std::env::var_os("DISPLAY").is_some_and(|s| !s.is_empty())
+		|| std::env::var_os("WAYLAND_DISPLAY").is_some_and(|s| !s.is_empty());
+	if flatscreenmode {
+		let mut plugin = WinitPlugin::<WakeUp>::default();
+		plugin.run_on_any_thread = true;
+		plugins = plugins
+			.add(plugin)
+			.disable::<ScheduleRunnerPlugin>()
+			.add(FlatscreenInputPlugin);
+	}
+	app.add_plugins(
+		if !args.flatscreen {
+			add_xr_plugins(plugins)
+				.set(OxrInitPlugin {
+					app_info: AppInfo {
+						name: "Stardust XR".into(),
+						version: bevy_mod_openxr::types::Version(0, 44, 1),
+					},
+					exts: {
+						// all OpenXR extensions can be requested here
+						let mut exts = OxrExtensions::default();
+						exts.enable_hand_tracking();
+						if args.overlay_priority.is_some() {
+							exts.enable_extx_overlay();
+						}
+						exts.khr_convert_timespec_time = true;
+						exts
+					},
+					..default()
+				})
+				.set(OxrRenderPlugin {
+					default_wait_frame: false,
+					..default()
+				})
+				.set(OxrReferenceSpacePlugin {
+					default_primary_ref_space: ReferenceSpaceType::LOCAL,
+				})
+				// Disable a bunch of unneeded plugins
+				// we don't do any action stuff that needs to integrate with the ecosystem
+				.disable::<OxrActionAttachingPlugin>()
+				.disable::<OxrActionSyncingPlugin>()
+		} else {
+			// enable a event
+			plugins = plugins.add(XrSessionPlugin { auto_handle: false });
+			bevy_dmabuf::wgpu_init::add_dmabuf_init_plugin(plugins)
+		}
+		.set(WindowPlugin {
+			primary_window: Some(Window {
+				transparent: args.transparent_flatscreen,
+				present_mode: PresentMode::AutoNoVsync,
+				composite_alpha_mode: if args.transparent_flatscreen {
+					CompositeAlphaMode::PreMultiplied
+				} else {
+					CompositeAlphaMode::Auto
+				},
+				title: "StardustXR server flatscreen mode".to_string(),
+				..default()
+			}),
+			..default()
+		}),
+	);
+
+	if flatscreenmode {
+		app.insert_resource(WinitSettings {
+			focused_mode: UpdateMode::Continuous,
+			unfocused_mode: UpdateMode::Continuous,
+		});
+	}
+
+	app.add_plugins(bevy_sk::hand::HandPlugin);
+	// app.add_plugins(HandGizmosPlugin);
+	app.world_mut().resource_mut::<AmbientLight>().brightness = 1000.0;
+	if let Some(priority) = args.overlay_priority {
+		app.insert_resource(OxrOverlaySettings {
+			session_layer_placement: priority,
+			..default()
+		});
+	}
+	app.insert_resource(OxrSessionConfig {
+		blend_mode_preference: vec![
+			EnvironmentBlendMode::ALPHA_BLEND,
+			EnvironmentBlendMode::ADDITIVE,
+			EnvironmentBlendMode::OPAQUE,
+		],
+		..default()
+	});
+	let mut pre_frame_wait = Schedule::new(PreFrameWait);
+	pre_frame_wait.set_executor_kind(ExecutorKind::MultiThreaded);
+	app.add_schedule(pre_frame_wait);
+	app.insert_resource(ClearColor(Color::BLACK.with_alpha(0.0)));
+	app.insert_resource(ObjectRegistryRes(object_registry));
+	#[cfg(feature = "bevy_debugging")]
+	{
+		use bevy::remote::{RemotePlugin, http::RemoteHttpPlugin};
+		app.add_plugins((RemotePlugin::default(), RemoteHttpPlugin::default()));
+	}
+	// the Stardust server plugins
+	// infra plugins
+	app.add_plugins(EntityHandlePlugin);
+	// node plugins
+	app.add_plugins((
+		SpatialNodePlugin,
+		ModelNodePlugin,
+		TextNodePlugin,
+		LinesNodePlugin,
+		AudioNodePlugin,
+	));
+	// object plugins
+	app.add_plugins((PlaySpacePlugin, HandPlugin, ControllerPlugin, HmdPlugin));
+	// feature plugins
+	app.add_plugins((WaylandPlugin, TrackingOffsetPlugin, FieldDebugGizmoPlugin));
+	app.add_systems(PostStartup, move || {
+		ready_notifier.notify_waiters();
+	});
+	app.add_observer(cam_observer);
+	app.add_systems(
+		XrFirst,
+		xr_step
+			.in_set(OxrWaitFrameSystem)
+			.in_set(XrHandleEvents::FrameLoop),
+	);
+
+	app.run()
+}
+
+fn cam_observer(
+	trigger: Trigger<OnAdd, Camera3d>,
+	mut query: Query<(&mut Projection, &mut Msaa), With<Camera3d>>,
+) {
+	let Ok((mut projection, mut msaa)) = query.get_mut(trigger.target()) else {
+		return;
+	};
+	info!("modifying cam");
+	match projection.deref_mut() {
+		Projection::Perspective(perspective_projection) => perspective_projection.near = 0.003,
+		Projection::Orthographic(orthographic_projection) => orthographic_projection.near = 0.003,
+		Projection::Custom(custom_projection) => {
+			if let Some(xr) = custom_projection.get_mut::<XrProjection>() {
+				xr.near = 0.003
+			} else {
+				error_once!("unknown custom camera projection");
+			}
+		}
+	}
+	*msaa = Msaa::Off;
+}
+
+fn xr_step(world: &mut World) {
+	// update things like the Xr input methods
+	world.run_schedule(PreFrameWait);
+	input::process_input();
+	let time = world.resource::<bevy::prelude::Time>().delta_secs_f64();
+	nodes::root::Root::send_frame_events(time);
+
+	// we are targeting the frame after the wait
+	if let Some(mut state) = world.get_resource_mut::<OxrFrameState>() {
+		state.predicted_display_time = openxr::Time::from_nanos(
+			state.predicted_display_time.as_nanos() + state.predicted_display_period.as_nanos(),
+		);
+	}
+
+	let should_wait = world
+		.run_system_cached(should_run_frame_loop)
+		.unwrap_or(false);
+	// we might want to do an adaptive sleep when not OpenXR waiting
+	if should_wait {
+		world.resource_scope::<OxrFrameWaiter, _>(|world, mut waiter| {
+			let state = waiter
+				.wait()
+				.inspect_err(|err| error!("failed to wait OpenXR frame: {err}"))
+				.ok();
+
+			if let Some(state) = state {
+				world.insert_resource(OxrFrameState(state));
+			}
+		});
+	}
+
+	tick_internal_client();
 }

src/nodes/alias.rs

@@ -1,38 +1,163 @@
-use crate::core::client::Client;
-
-use super::Node;
-use std::sync::{Arc, Weak};
+use super::{Aspect, AspectIdentifier, Node};
+use crate::core::{client::Client, error::Result, registry::Registry};
+use std::{
+	ops::Add,
+	sync::{Arc, Weak},
+};
 
 #[derive(Debug, Default, Clone)]
 pub struct AliasInfo {
-	pub(super) local_signals: Vec<&'static str>,
-	pub(super) local_methods: Vec<&'static str>,
-	pub(super) remote_signals: Vec<&'static str>,
+	pub(super) server_signals: Vec<u64>,
+	pub(super) server_methods: Vec<u64>,
+	pub(super) client_signals: Vec<u64>,
+}
+impl Add for AliasInfo {
+	type Output = AliasInfo;
+	fn add(mut self, mut rhs: Self) -> Self::Output {
+		self.server_signals.append(&mut rhs.server_signals);
+		self.server_methods.append(&mut rhs.server_methods);
+		self.client_signals.append(&mut rhs.client_signals);
+		self
+	}
 }
 
-#[allow(dead_code)]
+#[derive(Debug)]
 pub struct Alias {
 	pub(super) node: Weak<Node>,
-	pub original: Weak<Node>,
-
-	pub info: AliasInfo,
+	pub(super) original: Weak<Node>,
+	pub(super) info: AliasInfo,
 }
 impl Alias {
-	pub fn new(
-		client: &Arc<Client>,
-		parent: &str,
-		name: &str,
+	pub fn create(
 		original: &Arc<Node>,
+		client: &Arc<Client>,
 		info: AliasInfo,
-	) -> Arc<Node> {
-		let node = Node::create(client, parent, name, true).add_to_scenegraph();
+		list: Option<&AliasList>,
+	) -> Result<Arc<Node>> {
+		let node = Node::generate(client, true).add_to_scenegraph()?;
+		Self::add_to(&node, original, info)?;
+		if let Some(list) = list {
+			list.add(&node);
+		}
+		Ok(node)
+	}
+	pub fn create_with_id(
+		original: &Arc<Node>,
+		client: &Arc<Client>,
+		new_id: u64,
+		info: AliasInfo,
+		list: Option<&AliasList>,
+	) -> Result<Arc<Node>> {
+		let node = Node::from_id(client, new_id, true).add_to_scenegraph()?;
+		Self::add_to(&node, original, info)?;
+		if let Some(list) = list {
+			list.add(&node);
+		}
+		Ok(node)
+	}
+
+	fn add_to(new_node: &Arc<Node>, original: &Arc<Node>, info: AliasInfo) -> Result<()> {
 		let alias = Alias {
-			node: Arc::downgrade(&node),
+			node: Arc::downgrade(new_node),
 			original: Arc::downgrade(original),
 			info,
 		};
 		let alias = original.aliases.add(alias);
-		let _ = node.alias.set(alias);
-		node
+		new_node.add_aspect_raw(alias);
+		Ok(())
+	}
+}
+impl AspectIdentifier for Alias {
+	const ID: u64 = 0;
+}
+impl Aspect for Alias {
+	fn as_any(self: Arc<Self>) -> Arc<dyn std::any::Any + Send + Sync + 'static> {
+		self
+	}
+	fn run_signal(
+		&self,
+		_calling_client: Arc<Client>,
+		_node: Arc<Node>,
+		_signal: u64,
+		_message: super::Message,
+	) -> Result<(), stardust_xr::scenegraph::ScenegraphError> {
+		Ok(())
+	}
+	fn run_method(
+		&self,
+		_calling_client: Arc<Client>,
+		_node: Arc<Node>,
+		_method: u64,
+		_message: super::Message,
+		_response: crate::core::scenegraph::MethodResponseSender,
+	) {
+	}
+}
+
+pub fn get_original(node: Arc<Node>, stop_on_disabled: bool) -> Option<Arc<Node>> {
+	let Ok(alias) = node.get_aspect::<Alias>() else {
+		return Some(node);
+	};
+	if stop_on_disabled && !node.enabled() {
+		return None;
+	}
+	get_original(alias.original.upgrade()?, stop_on_disabled)
+}
+pub fn links_to(alias: Arc<Node>, original: Weak<Node>) -> bool {
+	let Ok(alias) = alias.get_aspect::<Alias>() else {
+		return false;
+	};
+	if alias.original.ptr_eq(&original) {
+		return true;
+	}
+	let Some(original_strong) = alias.original.upgrade() else {
+		return false;
+	};
+	links_to(original_strong, original)
+}
+
+#[derive(Debug, Default, Clone)]
+pub struct AliasList(Registry<Node>);
+impl AliasList {
+	fn add(&self, node: &Arc<Node>) {
+		self.0.add_raw(node);
+	}
+	pub fn get_from_original_node(&self, original: Weak<Node>) -> Option<Arc<Node>> {
+		self.0
+			.get_valid_contents()
+			.into_iter()
+			.find(move |node| links_to(node.clone(), original.clone()))
+	}
+	pub fn get_from_aspect<A: AspectIdentifier>(&self, aspect: &A) -> Option<Arc<Node>> {
+		self.0.get_valid_contents().into_iter().find(|node| {
+			let Some(node) = get_original(node.clone(), false) else {
+				return false;
+			};
+			let Ok(aspect2) = node.get_aspect::<A>() else {
+				return false;
+			};
+			std::ptr::eq(Arc::as_ptr(&aspect2), aspect)
+		})
+	}
+	pub fn get_aliases(&self) -> Vec<Arc<Node>> {
+		self.0.get_valid_contents()
+	}
+	pub fn remove_aspect<A: AspectIdentifier>(&self, aspect: &A) {
+		self.0.retain(|node| {
+			let Some(original) = get_original(node.clone(), false) else {
+				return false;
+			};
+			let Ok(aspect2) = original.get_aspect::<A>() else {
+				return false;
+			};
+			!std::ptr::eq(Arc::as_ptr(&aspect2), aspect)
+		})
+	}
+}
+impl Drop for AliasList {
+	fn drop(&mut self) {
+		for node in self.0.take_valid_contents() {
+			node.destroy();
+		}
 	}
 }

src/nodes/audio.rs

@@ -0,0 +1,168 @@
+use super::spatial::SpatialNode;
+use super::{Aspect, AspectIdentifier, Node};
+use crate::core::client::Client;
+use crate::core::entity_handle::EntityHandle;
+use crate::core::error::Result;
+use crate::core::registry::Registry;
+use crate::core::resource::get_resource_file;
+use crate::nodes::spatial::{SPATIAL_ASPECT_ALIAS_INFO, Spatial, Transform};
+use bevy::audio::{PlaybackMode, Volume};
+use bevy_mod_openxr::session::OxrSession;
+use bevy_mod_xr::session::{XrPreDestroySession, XrSessionCreated};
+use bevy_mod_xr::spaces::XrSpace;
+use color_eyre::eyre::eyre;
+use parking_lot::Mutex;
+use stardust_xr::values::ResourceID;
+
+use bevy::prelude::*;
+use bevy::transform::components::Transform as BevyTransform;
+use std::sync::{Arc, OnceLock};
+use std::{ffi::OsStr, path::PathBuf};
+
+pub struct AudioNodePlugin;
+impl Plugin for AudioNodePlugin {
+	fn build(&self, app: &mut App) {
+		app.add_systems(Update, update_sound_event);
+		app.add_systems(XrSessionCreated, spawn_hmd_audio_listener);
+		app.add_systems(XrPreDestroySession, despawn_hmd_audio_listener);
+	}
+}
+
+fn despawn_hmd_audio_listener(mut cmds: Commands, session: Res<OxrSession>, res: Res<HmdListener>) {
+	cmds.remove_resource::<HmdListener>();
+	cmds.entity(res.0).despawn();
+	_ = session.destroy_space(res.1);
+}
+
+fn spawn_hmd_audio_listener(mut cmds: Commands, session: Res<OxrSession>) {
+	let space = session
+		.create_reference_space(openxr::ReferenceSpaceType::VIEW, BevyTransform::IDENTITY)
+		.unwrap();
+	let listener = cmds
+		.spawn((
+			Name::new("HMD audio listener"),
+			space.0,
+			SpatialListener::new(0.2),
+		))
+		.id();
+	cmds.insert_resource(HmdListener(listener, space.0));
+}
+#[derive(Resource)]
+struct HmdListener(Entity, XrSpace);
+fn update_sound_event(
+	mut cmds: Commands,
+	sinks: Query<&SpatialAudioSink>,
+	asset_server: Res<AssetServer>,
+) {
+	for sound in SOUND_REGISTRY.get_valid_contents() {
+		if sound.entity.get().is_none() {
+			let handle = asset_server.load(sound.pending_audio_path.as_path());
+			sound
+				.entity
+				.set(
+					cmds.spawn((
+						Name::new("Audio Node"),
+						SpatialNode(Arc::downgrade(&sound.spatial)),
+						AudioPlayer::new(handle),
+						PlaybackSettings {
+							mode: PlaybackMode::Once,
+							volume: Volume::Linear(sound.volume),
+							speed: 1.0,
+							paused: true,
+							muted: false,
+							spatial: true,
+							spatial_scale: None,
+						},
+					))
+					.id()
+					.into(),
+				)
+				.unwrap();
+		}
+		if let Some(sink) = sound.entity.get().and_then(|e| sinks.get(e.0).ok()) {
+			if sound.play.lock().take().is_some() {
+				sink.play();
+			}
+			if sound.stop.lock().take().is_some() {
+				sink.stop();
+			}
+		}
+	}
+}
+
+static SOUND_REGISTRY: Registry<Sound> = Registry::new();
+
+stardust_xr_server_codegen::codegen_audio_protocol!();
+pub struct Sound {
+	spatial: Arc<Spatial>,
+
+	volume: f32,
+	pending_audio_path: PathBuf,
+	entity: OnceLock<EntityHandle>,
+	stop: Mutex<Option<()>>,
+	play: Mutex<Option<()>>,
+}
+impl Sound {
+	pub fn add_to(node: &Arc<Node>, resource_id: ResourceID) -> Result<Arc<Sound>> {
+		let pending_audio_path = get_resource_file(
+			&resource_id,
+			&*node.get_client().ok_or_else(|| eyre!("Client not found"))?,
+			&[OsStr::new("wav"), OsStr::new("mp3")],
+		)
+		.ok_or_else(|| eyre!("Resource not found"))?;
+		let sound = Sound {
+			spatial: node.get_aspect::<Spatial>().unwrap().clone(),
+			volume: 1.0,
+			pending_audio_path,
+			entity: OnceLock::new(),
+			stop: Mutex::new(None),
+			play: Mutex::new(None),
+		};
+		let sound_arc = SOUND_REGISTRY.add(sound);
+		node.add_aspect_raw(sound_arc.clone());
+		Ok(sound_arc)
+	}
+}
+impl AspectIdentifier for Sound {
+	impl_aspect_for_sound_aspect_id! {}
+}
+impl Aspect for Sound {
+	impl_aspect_for_sound_aspect! {}
+}
+impl SoundAspect for Sound {
+	fn play(node: Arc<Node>, _calling_client: Arc<Client>) -> Result<()> {
+		let sound = node.get_aspect::<Sound>().unwrap();
+		sound.play.lock().replace(());
+		Ok(())
+	}
+	fn stop(node: Arc<Node>, _calling_client: Arc<Client>) -> Result<()> {
+		let sound = node.get_aspect::<Sound>().unwrap();
+		sound.stop.lock().replace(());
+		Ok(())
+	}
+}
+impl Drop for Sound {
+	fn drop(&mut self) {
+		SOUND_REGISTRY.remove(self);
+	}
+}
+
+impl InterfaceAspect for Interface {
+	#[doc = "Create a sound node. WAV and MP3 are supported."]
+	fn create_sound(
+		_node: Arc<Node>,
+		calling_client: Arc<Client>,
+		id: u64,
+		parent: Arc<Node>,
+		transform: Transform,
+		resource: ResourceID,
+	) -> Result<()> {
+		let node = Node::from_id(&calling_client, id, true);
+		let parent = parent.get_aspect::<Spatial>()?;
+		let transform = transform.to_mat4(true, true, true);
+		let node = node.add_to_scenegraph()?;
+		Spatial::add_to(&node, Some(parent.clone()), transform, false);
+		Sound::add_to(&node, resource)?;
+		Ok(())
+	}
+}

src/nodes/data.rs

@@ -1,293 +0,0 @@
-use super::alias::AliasInfo;
-use super::fields::Field;
-use super::spatial::{parse_transform, Spatial};
-use super::{Alias, Node};
-use crate::core::client::Client;
-use crate::core::nodelist::LifeLinkedNodeList;
-use crate::core::registry::Registry;
-use crate::nodes::fields::find_field;
-use crate::nodes::spatial::find_spatial_parent;
-use anyhow::{anyhow, ensure, Result};
-use glam::vec3a;
-use parking_lot::Mutex;
-use serde::Deserialize;
-use stardust_xr::schemas::flex::{deserialize, serialize};
-use stardust_xr::values::Transform;
-use std::sync::{Arc, Weak};
-
-static PULSE_SENDER_REGISTRY: Registry<PulseSender> = Registry::new();
-static PULSE_RECEIVER_REGISTRY: Registry<PulseReceiver> = Registry::new();
-
-fn mask_matches(mask_map_lesser: &Mask, mask_map_greater: &Mask) -> bool {
-	(|| -> Result<_> {
-		for key in mask_map_lesser.get_mask()?.iter_keys() {
-			let lesser_key_type = mask_map_lesser.get_mask()?.index(key)?.flexbuffer_type();
-			let greater_key_type = mask_map_greater.get_mask()?.index(key)?.flexbuffer_type();
-			if lesser_key_type != greater_key_type {
-				return Err(flexbuffers::ReaderError::InvalidPackedType {}.into());
-			}
-		}
-		Ok(())
-	})()
-	.is_ok()
-}
-
-type MaskMapGetFn = fn(&[u8]) -> Result<flexbuffers::MapReader<&[u8]>>;
-pub struct Mask {
-	binary: Vec<u8>,
-	get_fn: MaskMapGetFn,
-}
-impl Mask {
-	pub fn get_mask(&self) -> Result<flexbuffers::MapReader<&[u8]>> {
-		(self.get_fn)(self.binary.as_slice())
-	}
-	pub fn set_mask(&mut self, binary: Vec<u8>, get_fn: MaskMapGetFn) {
-		self.binary = binary;
-		self.get_fn = get_fn;
-	}
-}
-impl Default for Mask {
-	fn default() -> Self {
-		Mask {
-			binary: Default::default(),
-			get_fn: mask_get_err,
-		}
-	}
-}
-fn mask_get_err(_binary: &[u8]) -> Result<flexbuffers::MapReader<&[u8]>> {
-	Err(anyhow!("You need to call setMask to set the mask!"))
-}
-fn mask_get_map_at_root(binary: &[u8]) -> Result<flexbuffers::MapReader<&[u8]>> {
-	flexbuffers::Reader::get_root(binary)
-		.map_err(|_| anyhow!("Mask is not a valid flexbuffer"))?
-		.get_map()
-		.map_err(|_| anyhow!("Mask is not a valid map"))
-}
-
-#[derive(Default)]
-pub struct PulseSender {
-	mask: Mutex<Mask>,
-	aliases: LifeLinkedNodeList,
-}
-impl PulseSender {
-	pub fn add_to(node: &Arc<Node>) -> Result<()> {
-		ensure!(
-			node.spatial.get().is_some(),
-			"Internal: Node does not have a spatial attached!"
-		);
-
-		let sender = Default::default();
-		let sender = PULSE_SENDER_REGISTRY.add(sender);
-		let _ = node.pulse_sender.set(sender);
-		node.add_local_signal("setMask", PulseSender::set_mask_flex);
-		node.add_local_method("getReceivers", PulseSender::get_receivers_flex);
-		Ok(())
-	}
-	pub fn set_mask_flex(node: &Node, _calling_client: Arc<Client>, data: &[u8]) -> Result<()> {
-		ensure!(
-			node.pulse_sender.get().is_some(),
-			"Internal: Node does not have a pulse sender aspect"
-		);
-		node.pulse_sender
-			.get()
-			.unwrap()
-			.mask
-			.lock()
-			.set_mask(data.to_vec(), mask_get_map_at_root);
-		Ok(())
-	}
-	fn get_receivers_flex(
-		node: &Node,
-		calling_client: Arc<Client>,
-		_data: &[u8],
-	) -> Result<Vec<u8>> {
-		let sender_spatial = node
-			.spatial
-			.get()
-			.ok_or_else(|| anyhow!("Node does not have a spatial aspect!"))?;
-		let sender = node
-			.pulse_sender
-			.get()
-			.ok_or_else(|| anyhow!("Node does not have a sender aspect!"))?;
-		let valid_receivers = PULSE_RECEIVER_REGISTRY.get_valid_contents();
-		let mut distance_sorted_receivers: Vec<(f32, &PulseReceiver)> = valid_receivers
-			.iter()
-			.filter(|receiver| receiver.get_field().is_some())
-			.filter(|receiver| mask_matches(&*sender.mask.lock(), &*receiver.mask.lock()))
-			.map(|receiver| {
-				(
-					receiver
-						.get_field()
-						.unwrap()
-						.distance(sender_spatial, vec3a(0_f32, 0_f32, 0_f32)),
-					receiver.as_ref(),
-				)
-			})
-			.collect();
-		distance_sorted_receivers.sort_by(|(d1, _), (d2, _)| d1.partial_cmp(d2).unwrap());
-		sender.aliases.clear();
-		let uids: Vec<String> = distance_sorted_receivers
-			.into_iter()
-			.map(|(_, receiver)| {
-				let receiver_alias = Alias::new(
-					&calling_client,
-					node.get_path(),
-					receiver.uid.as_str(),
-					receiver.node.upgrade().as_ref().unwrap(),
-					AliasInfo {
-						local_methods: vec!["sendData"],
-						..Default::default()
-					},
-				);
-				sender.aliases.add(Arc::downgrade(&receiver_alias));
-
-				receiver.uid.clone()
-			})
-			.collect();
-
-		serialize(uids).map_err(|e| e.into())
-	}
-}
-impl Drop for PulseSender {
-	fn drop(&mut self) {
-		PULSE_SENDER_REGISTRY.remove(self);
-	}
-}
-
-pub struct PulseReceiver {
-	uid: String,
-	node: Weak<Node>,
-	pub mask: Mutex<Mask>,
-	field: Weak<Field>,
-}
-impl PulseReceiver {
-	pub fn add_to(node: &Arc<Node>, field: Arc<Field>) -> Result<()> {
-		ensure!(
-			node.spatial.get().is_some(),
-			"Internal: Node does not have a spatial attached!"
-		);
-
-		let receiver = PulseReceiver {
-			uid: node.uid.clone(),
-			node: Arc::downgrade(node),
-			field: Arc::downgrade(&field),
-			mask: Default::default(),
-		};
-		let receiver = PULSE_RECEIVER_REGISTRY.add(receiver);
-		let _ = node.pulse_receiver.set(receiver);
-		node.add_local_signal("setMask", PulseReceiver::set_mask_flex);
-		node.add_local_signal("sendData", PulseReceiver::send_data_flex);
-		Ok(())
-	}
-	fn get_field(&self) -> Option<Arc<Field>> {
-		self.field.upgrade()
-	}
-	fn send_data_flex(node: &Node, _calling_client: Arc<Client>, data: &[u8]) -> Result<()> {
-		ensure!(
-			node.pulse_receiver.get().is_some(),
-			"Internal: Node does not have a pulse receiver aspect"
-		);
-		let receiver_mask = node.pulse_receiver.get().unwrap().mask.lock();
-		let data_mask = Mask {
-			binary: data.to_vec(),
-			get_fn: mask_get_map_at_root,
-		};
-		if !mask_matches(&receiver_mask, &data_mask) {
-			return Err(anyhow!(
-				"Message does not contain the same keys as the receiver mask"
-			));
-		}
-		drop(receiver_mask);
-		node.send_remote_signal("pulse", data)?;
-		Ok(())
-	}
-	fn set_mask_flex(node: &Node, _calling_client: Arc<Client>, data: &[u8]) -> Result<()> {
-		ensure!(
-			node.pulse_receiver.get().is_some(),
-			"Internal: Node does not have a pulse receiver aspect"
-		);
-		node.pulse_receiver
-			.get()
-			.unwrap()
-			.mask
-			.lock()
-			.set_mask(data.to_vec(), mask_get_map_at_root);
-		Ok(())
-	}
-}
-
-impl Drop for PulseReceiver {
-	fn drop(&mut self) {
-		PULSE_RECEIVER_REGISTRY.remove(self);
-	}
-}
-
-pub fn create_interface(client: &Arc<Client>) {
-	let node = Node::create(client, "", "data", false);
-	node.add_local_signal("createPulseSender", create_pulse_sender_flex);
-	node.add_local_signal("createPulseReceiver", create_pulse_receiver_flex);
-	node.add_to_scenegraph();
-}
-
-// pub fn mask_get_map_pulse_sender_create_args(mask: &Mask) -> Result<flexbuffers::MapReader<&[u8]>> {
-// 	flexbuffers::Reader::get_root(mask.binary.as_slice())
-// 		.map_err(|_| anyhow!("Mask is not a valid flexbuffer"))?
-// 		.get_vector()?
-// 		.index(4)?
-// 		.get_map()
-// 		.map_err(|_| anyhow!("Mask is not a valid map"))
-// }
-pub fn create_pulse_sender_flex(
-	_node: &Node,
-	calling_client: Arc<Client>,
-	data: &[u8],
-) -> Result<()> {
-	#[derive(Deserialize)]
-	struct CreatePulseSenderInfo<'a> {
-		name: &'a str,
-		parent_path: &'a str,
-		transform: Transform,
-	}
-	let info: CreatePulseSenderInfo = deserialize(data)?;
-	let node = Node::create(&calling_client, "/data/sender", info.name, true);
-	let parent = find_spatial_parent(&calling_client, info.parent_path)?;
-	let transform = parse_transform(info.transform, true, true, false)?;
-	let node = node.add_to_scenegraph();
-	Spatial::add_to(&node, Some(parent), transform)?;
-	PulseSender::add_to(&node)?;
-	Ok(())
-}
-
-// pub fn mask_get_map_pulse_receiver_create_args(
-// 	mask: &Mask,
-// ) -> Result<flexbuffers::MapReader<&[u8]>> {
-// 	flexbuffers::Reader::get_root(mask.binary.as_slice())
-// 		.map_err(|_| anyhow!("Mask is not a valid flexbuffer"))?
-// 		.get_vector()?
-// 		.index(5)?
-// 		.get_map()
-// 		.map_err(|_| anyhow!("Mask is not a valid map"))
-// }
-pub fn create_pulse_receiver_flex(
-	_node: &Node,
-	calling_client: Arc<Client>,
-	data: &[u8],
-) -> Result<()> {
-	#[derive(Deserialize)]
-	struct CreatePulseReceiverInfo<'a> {
-		name: &'a str,
-		parent_path: &'a str,
-		transform: Transform,
-		field_path: &'a str,
-	}
-	let info: CreatePulseReceiverInfo = deserialize(data)?;
-	let node = Node::create(&calling_client, "/data/sender", info.name, true);
-	let parent = find_spatial_parent(&calling_client, info.parent_path)?;
-	let transform = parse_transform(info.transform, true, true, false)?;
-	let field = find_field(&calling_client, info.field_path)?;
-
-	let node = node.add_to_scenegraph();
-	Spatial::add_to(&node, Some(parent), transform)?;
-	PulseReceiver::add_to(&node, field)?;
-	Ok(())
-}

src/nodes/drawable/holdout.wgsl

@@ -0,0 +1,5 @@
+fn fragment(
+    in: VertexOutput
+) -> FragmentOutput {
+    return vec4(0.0);
+}

src/nodes/drawable/lines.rs

@@ -0,0 +1,286 @@
+use super::{Line, LinesAspect};
+use crate::{
+	BevyMaterial,
+	core::{
+		client::Client, color::ColorConvert, entity_handle::EntityHandle, error::Result,
+		registry::Registry,
+	},
+	nodes::{
+		Node,
+		spatial::{Spatial, SpatialNode},
+	},
+};
+use bevy::{
+	asset::RenderAssetUsages,
+	prelude::*,
+	render::{
+		mesh::{Indices, MeshAabb, PrimitiveTopology},
+		primitives::Aabb,
+	},
+};
+use glam::Vec3;
+use parking_lot::Mutex;
+use std::sync::{
+	Arc, OnceLock,
+	atomic::{AtomicBool, Ordering},
+};
+
+pub struct LinesNodePlugin;
+
+impl Plugin for LinesNodePlugin {
+	fn build(&self, app: &mut App) {
+		app.add_systems(Update, build_line_mesh);
+	}
+}
+
+fn build_line_mesh(
+	mut meshes: ResMut<Assets<Mesh>>,
+	mut cmds: Commands,
+	mut materials: ResMut<Assets<BevyMaterial>>,
+) {
+	for lines in LINES_REGISTRY
+		.get_valid_contents()
+		.into_iter()
+		.filter(|l| l.gen_mesh.load(Ordering::Relaxed))
+	{
+		lines.gen_mesh.store(false, Ordering::Relaxed);
+		let mut vertex_positions = Vec::<Vec3>::new();
+		let mut vertex_normals = Vec::<Vec3>::new();
+		let mut vertex_colors = Vec::<[f32; 4]>::new();
+		let mut vertex_indices = Vec::<u32>::new();
+		let lines_data = lines.data.lock();
+		if lines_data.is_empty() {
+			*lines.bounds.lock() = Aabb::default();
+			match lines.entity.get() {
+				Some(e) => cmds.entity(**e),
+				None => {
+					let e = cmds.spawn((
+						Name::new("LinesNode"),
+						SpatialNode(Arc::downgrade(&lines.spatial)),
+					));
+					_ = lines.entity.set(e.id().into());
+					e
+				}
+			}
+			.remove::<Mesh3d>();
+			continue;
+		}
+
+		let mut indices_set = 0;
+		for line in lines_data.iter() {
+			let start_set = indices_set;
+			// Create a sliding window of points to process each segment of the line
+			// For cyclic lines: wraps around by connecting last point back to first
+			// For non-cyclic lines: handles endpoints with None values
+			let point_windows = {
+				let mut out = Vec::new();
+				let mut last = line.cyclic.then(|| line.points.last()).flatten();
+				let mut peekable = line.points.iter().peekable();
+				while let Some(curr) = peekable.next() {
+					// Skip this point if it has the same position as the previous point
+					if let Some(prev) = last {
+						if Vec3::from(prev.point) == Vec3::from(curr.point) {
+							last = Some(curr);
+							continue;
+						}
+					}
+
+					let mut end = false;
+					// Determine the next point - either the next in sequence or
+					// for cyclic lines, wrap back to first point at the end
+					let next = match peekable.peek() {
+						Some(v) => Some(*v),
+						None => {
+							end = true;
+							line.cyclic.then(|| line.points.first()).flatten()
+						}
+					};
+
+					out.push((last, curr, next, end));
+					last = Some(curr);
+				}
+				out
+			};
+			// if we can't make a full line, don't bother trying
+			if point_windows.len() < 2 {
+				continue;
+			}
+			for (last, curr, next, last_point) in point_windows {
+				let last_quat = last.map(|v| {
+					Quat::from_rotation_arc(
+						Vec3::Y,
+						(Vec3::from(curr.point) - Vec3::from(v.point)).normalize(),
+					)
+				});
+				let next_quat = next.map(|v| {
+					Quat::from_rotation_arc(
+						Vec3::Y,
+						(Vec3::from(v.point) - Vec3::from(curr.point)).normalize(),
+					)
+				});
+				let quat = match (last_quat, next_quat) {
+					(None, None) => {
+						error!("no previous or next point in line");
+						break;
+					}
+					(None, Some(next)) => next,
+					(Some(last), None) => last,
+					(Some(last), Some(next)) => last.lerp(next, 0.5),
+				};
+				let normals = [
+					Vec3::X,
+					Vec3::new(1., 0., 1.).normalize(),
+					Vec3::Z,
+					Vec3::new(-1., 0., 1.).normalize(),
+					Vec3::NEG_X,
+					Vec3::new(-1., 0., -1.).normalize(),
+					Vec3::NEG_Z,
+					Vec3::new(1., 0., -1.).normalize(),
+				]
+				.map(Vec3::normalize)
+				.map(|v| (quat * v));
+				let points = normals.map(|v| (v * curr.thickness) + Vec3::from(curr.point));
+				vertex_normals.extend(normals);
+				vertex_positions.extend(points);
+				vertex_colors.extend([curr.color.to_bevy().to_srgba().to_f32_array(); 8]);
+				// Only connect vertices between segments if this isn't the end point
+				if !last_point {
+					vertex_indices.extend(indices(indices_set));
+				}
+				indices_set += 1;
+			}
+			// Handle the connection between start and end points:
+			// - For cyclic lines: connect last segment back to first
+			// - For non-cyclic lines: add caps at both ends
+			if line.cyclic {
+				vertex_indices.extend(cyclic_indices(start_set, indices_set - 1));
+			} else {
+				vertex_indices.extend(cap_indices(start_set, false));
+				vertex_indices.extend(cap_indices(indices_set - 1, true));
+			}
+		}
+		let mut mesh = Mesh::new(
+			PrimitiveTopology::TriangleList,
+			RenderAssetUsages::RENDER_WORLD,
+		);
+		if vertex_colors.iter().flatten().any(|v| !v.is_finite()) {
+			panic!("vertex colors contains non finite float: {vertex_colors:#?}",);
+		}
+		if vertex_normals.iter().any(|v| !v.is_finite()) {
+			panic!("normals contains non finite dir: {vertex_normals:#?}",);
+		}
+		if vertex_normals.iter().any(|v| !v.is_normalized()) {
+			panic!("normals contains non normalized dir: {vertex_normals:#?}",);
+		}
+		if vertex_positions.iter().any(|v| !v.is_finite()) {
+			panic!("vertex positions contains non finite pos: {vertex_positions:#?}",);
+		}
+		mesh.insert_indices(Indices::U32(vertex_indices));
+		mesh.insert_attribute(Mesh::ATTRIBUTE_COLOR, vertex_colors);
+		mesh.insert_attribute(Mesh::ATTRIBUTE_NORMAL, vertex_normals);
+		mesh.insert_attribute(Mesh::ATTRIBUTE_POSITION, vertex_positions.clone());
+		if let Some(aabb) = mesh.compute_aabb() {
+			info!(?aabb);
+			*lines.bounds.lock() = aabb;
+		}
+
+		match lines.entity.get() {
+			Some(e) => cmds.entity(**e),
+			None => {
+				let e = cmds.spawn((
+					Name::new("LinesNode"),
+					SpatialNode(Arc::downgrade(&lines.spatial)),
+					MeshMaterial3d(materials.add(BevyMaterial {
+						base_color: Color::WHITE,
+						perceptual_roughness: 1.0,
+						// TODO: this should be Blend
+						alpha_mode: AlphaMode::Opaque,
+						..default()
+					})),
+				));
+				_ = lines.entity.set(e.id().into());
+				e
+			}
+		}
+		.insert(Mesh3d(meshes.add(mesh)));
+	}
+}
+
+const END_CAP_INDICES: [u32; 18] = [0, 1, 7, 7, 1, 2, 7, 2, 6, 6, 2, 3, 6, 3, 5, 5, 3, 4];
+fn cap_indices(set: u32, flip: bool) -> [u32; END_CAP_INDICES.len()] {
+	let mut out = END_CAP_INDICES.map(|v| v + (set * 8));
+	if flip {
+		out.reverse();
+	}
+	out
+}
+
+// const BASE: [u16; 6] = [0, 8, 1, 8, 9, 1];
+
+// Defines how vertices are connected between consecutive cross-sections to form the tube
+const INDICES: [u32; 48] = [
+	0, 8, 1, 8, 9, 1, 1, 9, 2, 9, 10, 2, 2, 10, 3, 10, 11, 3, 3, 11, 4, 11, 12, 4, 4, 12, 5, 12,
+	13, 5, 5, 13, 6, 13, 14, 6, 6, 14, 7, 14, 15, 7, 7, 15, 0, 15, 8, 0,
+];
+fn indices(set: u32) -> [u32; INDICES.len()] {
+	INDICES.map(|v| v + (set * 8))
+}
+fn cyclic_indices(start_set: u32, end_set: u32) -> [u32; INDICES.len()] {
+	let mut out = INDICES.map(|v| {
+		if v < 8 {
+			v + ((start_set) * 8)
+		} else {
+			v + ((end_set - 1) * 8)
+		}
+	});
+	out.reverse();
+	out
+}
+
+static LINES_REGISTRY: Registry<Lines> = Registry::new();
+
+pub struct Lines {
+	spatial: Arc<Spatial>,
+	data: Mutex<Vec<Line>>,
+	gen_mesh: AtomicBool,
+	entity: OnceLock<EntityHandle>,
+	bounds: Mutex<Aabb>,
+}
+impl Lines {
+	pub fn add_to(node: &Arc<Node>, lines: Vec<Line>) -> Result<Arc<Lines>> {
+		let _ = node
+			.get_aspect::<Spatial>()
+			.unwrap()
+			.bounding_box_calc
+			.set(|node| {
+				node.get_aspect::<Lines>()
+					.ok()
+					.map(|v| *v.bounds.lock())
+					.unwrap_or_default()
+			});
+
+		let lines = LINES_REGISTRY.add(Lines {
+			spatial: node.get_aspect::<Spatial>()?.clone(),
+			data: Mutex::new(lines),
+			gen_mesh: AtomicBool::new(true),
+			entity: OnceLock::new(),
+			bounds: Mutex::new(Aabb::default()),
+		});
+		node.add_aspect_raw(lines.clone());
+
+		Ok(lines)
+	}
+}
+impl LinesAspect for Lines {
+	fn set_lines(node: Arc<Node>, _calling_client: Arc<Client>, lines: Vec<Line>) -> Result<()> {
+		let lines_aspect = node.get_aspect::<Lines>()?;
+		*lines_aspect.data.lock() = lines;
+		lines_aspect.gen_mesh.store(true, Ordering::Relaxed);
+		Ok(())
+	}
+}
+impl Drop for Lines {
+	fn drop(&mut self) {
+		LINES_REGISTRY.remove(self);
+	}
+}

src/nodes/drawable/mod.rs

@@ -1,69 +1,132 @@
+pub mod lines;
 pub mod model;
 pub mod text;
 
-use super::Node;
-use crate::core::client::Client;
-use anyhow::Result;
+use self::{lines::Lines, model::Model, text::Text};
+use super::{
+	Aspect, AspectIdentifier, Node,
+	spatial::{Spatial, Transform},
+};
+use crate::core::{client::Client, error::Result, resource::get_resource_file};
+use crate::nodes::spatial::SPATIAL_ASPECT_ALIAS_INFO;
+use color_eyre::eyre::eyre;
+use model::ModelPart;
 use parking_lot::Mutex;
-use serde::Deserialize;
-use stardust_xr::schemas::flex::deserialize;
-use std::{path::PathBuf, sync::Arc};
-use stereokit::{lifecycle::DrawContext, texture::Texture, StereoKit};
+use stardust_xr::values::ResourceID;
+use std::{ffi::OsStr, path::PathBuf, sync::Arc};
 
-pub fn create_interface(client: &Arc<Client>) {
-	let node = Node::create(client, "", "drawable", false);
-	node.add_local_signal("createModel", model::create_flex);
-	node.add_local_signal("createText", text::create_flex);
-	node.add_local_signal("setSkyFile", set_sky_file_flex);
-	node.add_to_scenegraph();
+static QUEUED_SKYLIGHT: Mutex<Option<Option<PathBuf>>> = Mutex::new(None);
+static QUEUED_SKYTEX: Mutex<Option<Option<PathBuf>>> = Mutex::new(None);
+
+stardust_xr_server_codegen::codegen_drawable_protocol!();
+
+impl AspectIdentifier for Lines {
+	impl_aspect_for_lines_aspect_id! {}
+}
+impl Aspect for Lines {
+	impl_aspect_for_lines_aspect! {}
+}
+impl AspectIdentifier for Model {
+	impl_aspect_for_model_aspect_id! {}
+}
+impl Aspect for Model {
+	impl_aspect_for_model_aspect! {}
+}
+impl AspectIdentifier for ModelPart {
+	impl_aspect_for_model_part_aspect_id! {}
+}
+impl Aspect for ModelPart {
+	impl_aspect_for_model_part_aspect! {}
+}
+impl AspectIdentifier for Text {
+	impl_aspect_for_text_aspect_id! {}
+}
+impl Aspect for Text {
+	impl_aspect_for_text_aspect! {}
 }
 
-pub fn draw(sk: &mut StereoKit, draw_ctx: &DrawContext) {
-	model::draw_all(sk, draw_ctx);
-	text::draw_all(sk, draw_ctx);
-
-	let new_skytex = QUEUED_SKYTEX.lock().take();
-	let mut new_skylight = QUEUED_SKYLIGHT.lock().take();
-	let same_file = new_skytex == new_skylight;
-
-	if let Some(skytex) = new_skytex {
-		if let Some((skytex, skylight)) =
-			Texture::from_cubemap_equirectangular(sk, &skytex, true, i32::MAX)
-		{
-			sk.set_skytex(&skytex);
-			if same_file {
-				sk.set_skylight(&skylight);
-				new_skylight = None;
-			}
-		}
+impl InterfaceAspect for Interface {
+	fn set_sky_tex(
+		_node: Arc<Node>,
+		calling_client: Arc<Client>,
+		tex: Option<ResourceID>,
+	) -> Result<()> {
+		let resource_path = tex
+			.map(|tex| {
+				get_resource_file(&tex, &calling_client, &[OsStr::new("hdr")])
+					.ok_or(eyre!("Could not find resource"))
+			})
+			.transpose()?;
+		QUEUED_SKYTEX.lock().replace(resource_path);
+		Ok(())
 	}
-	if let Some(skylight) = new_skylight {
-		if let Some((_, skylight)) =
-			Texture::from_cubemap_equirectangular(sk, &skylight, true, i32::MAX)
-		{
-			sk.set_skylight(&skylight);
-		}
+
+	fn set_sky_light(
+		_node: Arc<Node>,
+		calling_client: Arc<Client>,
+		light: Option<ResourceID>,
+	) -> Result<()> {
+		let resource_path = light
+			.map(|light| {
+				get_resource_file(&light, &calling_client, &[OsStr::new("hdr")])
+					.ok_or(eyre!("Could not find resource"))
+			})
+			.transpose()?;
+		QUEUED_SKYLIGHT.lock().replace(resource_path);
+		Ok(())
+	}
+
+	fn create_lines(
+		_node: Arc<Node>,
+		calling_client: Arc<Client>,
+		id: u64,
+		parent: Arc<Node>,
+		transform: Transform,
+		lines: Vec<Line>,
+	) -> Result<()> {
+		let node = Node::from_id(&calling_client, id, true);
+		let parent = parent.get_aspect::<Spatial>()?;
+		let transform = transform.to_mat4(true, true, true);
+
+		let node = node.add_to_scenegraph()?;
+		Spatial::add_to(&node, Some(parent.clone()), transform, false);
+		Lines::add_to(&node, lines)?;
+		Ok(())
+	}
+
+	fn load_model(
+		_node: Arc<Node>,
+		calling_client: Arc<Client>,
+		id: u64,
+		parent: Arc<Node>,
+		transform: Transform,
+		model: ResourceID,
+	) -> Result<()> {
+		let node = Node::from_id(&calling_client, id, true);
+		let parent = parent.get_aspect::<Spatial>()?;
+		let transform = transform.to_mat4(true, true, true);
+		let node = node.add_to_scenegraph()?;
+		Spatial::add_to(&node, Some(parent.clone()), transform, false);
+		Model::add_to(&node, model)?;
+		Ok(())
+	}
+
+	fn create_text(
+		_node: Arc<Node>,
+		calling_client: Arc<Client>,
+		id: u64,
+		parent: Arc<Node>,
+		transform: Transform,
+		text: String,
+		style: TextStyle,
+	) -> Result<()> {
+		let node = Node::from_id(&calling_client, id, true);
+		let parent = parent.get_aspect::<Spatial>()?;
+		let transform = transform.to_mat4(true, true, true);
+
+		let node = node.add_to_scenegraph()?;
+		Spatial::add_to(&node, Some(parent.clone()), transform, false);
+		Text::add_to(&node, text, style)?;
+		Ok(())
 	}
 }
-
-static QUEUED_SKYLIGHT: Mutex<Option<PathBuf>> = Mutex::new(None);
-static QUEUED_SKYTEX: Mutex<Option<PathBuf>> = Mutex::new(None);
-
-fn set_sky_file_flex(_node: &Node, _calling_client: Arc<Client>, data: &[u8]) -> Result<()> {
-	#[derive(Deserialize)]
-	struct SkyFileInfo {
-		path: PathBuf,
-		skytex: Option<bool>,
-		skylight: Option<bool>,
-	}
-	let info: SkyFileInfo = deserialize(data)?;
-	info.path.metadata()?;
-	if info.skytex.unwrap_or_default() {
-		QUEUED_SKYTEX.lock().replace(info.path.clone());
-	}
-	if info.skylight.unwrap_or_default() {
-		QUEUED_SKYLIGHT.lock().replace(info.path);
-	}
-
-	Ok(())
-}

src/nodes/drawable/model.rs

@@ -1,184 +1,625 @@
-use super::Node;
+use super::{MODEL_PART_ASPECT_ALIAS_INFO, MaterialParameter, ModelAspect, ModelPartAspect};
+use crate::core::bevy_channel::{BevyChannel, BevyChannelReader};
 use crate::core::client::Client;
-use crate::core::destroy_queue;
+use crate::core::color::ColorConvert as _;
+use crate::core::entity_handle::EntityHandle;
+use crate::core::error::Result;
 use crate::core::registry::Registry;
-use crate::core::resource::ResourceID;
-use crate::nodes::spatial::{find_spatial_parent, parse_transform, Spatial};
-use anyhow::{anyhow, ensure, Result};
-use once_cell::sync::OnceCell;
+use crate::core::resource::get_resource_file;
+use crate::nodes::Node;
+use crate::nodes::alias::{Alias, AliasList};
+use crate::nodes::spatial::{Spatial, SpatialNode};
+use crate::{BevyMaterial, bail};
+use bevy::asset::{load_internal_asset, weak_handle};
+use bevy::gltf::GltfLoaderSettings;
+use bevy::pbr::{ExtendedMaterial, MaterialExtension};
+use bevy::prelude::*;
+use bevy::render::primitives::Aabb;
+use bevy::render::render_resource::{AsBindGroup, ShaderRef};
+use color_eyre::eyre::eyre;
 use parking_lot::Mutex;
-use prisma::{Rgb, Rgba};
-use rustc_hash::FxHashMap;
-use send_wrapper::SendWrapper;
-use serde::Deserialize;
-use stardust_xr::schemas::flex::deserialize;
-use stardust_xr::values::Transform;
-use std::fmt::Error;
+use rustc_hash::{FxHashMap, FxHasher};
+use stardust_xr::values::ResourceID;
+use std::ffi::OsStr;
+use std::hash::{Hash, Hasher};
 use std::path::PathBuf;
-use std::sync::Arc;
-use stereokit::lifecycle::DrawContext;
-use stereokit::material::Material;
-use stereokit::model::Model as SKModel;
-use stereokit::render::RenderLayer;
-use stereokit::texture::Texture;
-use stereokit::StereoKit;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::{Arc, OnceLock, Weak};
 
-static MODEL_REGISTRY: Registry<Model> = Registry::new();
+static LOAD_MODEL: BevyChannel<(Arc<Model>, PathBuf)> = BevyChannel::new();
 
-#[derive(Deserialize, Debug)]
-#[serde(untagged)]
-pub enum MaterialParameter {
-	Texture(PathBuf),
+type HoldoutMaterial = ExtendedMaterial<BevyMaterial, HoldoutExtension>;
+const HOLDOUT_SHADER_HANDLE: Handle<Shader> = weak_handle!("92b481b7-d3da-4188-b252-2335ec814ee2");
+const HOLDOUT_MATERIAL_HANDLE: Handle<HoldoutMaterial> =
+	weak_handle!("d56f1d62-9121-434b-a34f-9f0bbd6b3390");
+
+pub struct ModelNodePlugin;
+#[derive(SystemSet, Hash, Debug, PartialEq, Eq, Clone, Copy)]
+pub struct ModelNodeSystemSet;
+impl Plugin for ModelNodePlugin {
+	fn build(&self, app: &mut App) {
+		LOAD_MODEL.init(app);
+
+		load_internal_asset!(
+			app,
+			HOLDOUT_SHADER_HANDLE,
+			"holdout.wgsl",
+			Shader::from_wgsl
+		);
+		app.add_plugins(MaterialPlugin::<HoldoutMaterial>::default());
+		app.world_mut()
+			.resource_mut::<Assets<HoldoutMaterial>>()
+			.insert(&HOLDOUT_MATERIAL_HANDLE, HoldoutMaterial::default());
+
+		app.init_resource::<MaterialRegistry>();
+		app.add_systems(
+			Update,
+			(
+				load_models,
+				gen_model_parts.after(TransformSystem::TransformPropagate),
+				apply_materials,
+			)
+				.chain()
+				.in_set(ModelNodeSystemSet),
+		);
+	}
 }
 
-pub struct Model {
-	space: Arc<Spatial>,
-	resource_id: ResourceID,
-	pending_model_path: OnceCell<PathBuf>,
-	pending_material_parameters: Mutex<FxHashMap<(u32, String), MaterialParameter>>,
-	pub pending_material_replacements: Mutex<FxHashMap<u32, Arc<SendWrapper<Material>>>>,
-	sk_model: OnceCell<SendWrapper<SKModel>>,
-}
-
-impl Model {
-	pub fn add_to(node: &Arc<Node>, resource_id: ResourceID) -> Result<Arc<Model>> {
-		ensure!(
-			node.spatial.get().is_some(),
-			"Internal: Node does not have a spatial attached!"
-		);
-		ensure!(
-			node.model.get().is_none(),
-			"Internal: Node already has a model attached!"
-		);
-		let model = Model {
-			space: node.spatial.get().unwrap().clone(),
-			resource_id,
-			pending_model_path: OnceCell::new(),
-			pending_material_parameters: Mutex::new(FxHashMap::default()),
-			pending_material_replacements: Mutex::new(FxHashMap::default()),
-			sk_model: OnceCell::new(),
-		};
-		node.add_local_signal("setMaterialParameter", Model::set_material_parameter);
-		let model_arc = MODEL_REGISTRY.add(model);
-		let _ = model_arc.pending_model_path.set(
-			model_arc
-				.resource_id
-				.get_file(
-					&node
-						.get_client()
-						.ok_or_else(|| anyhow!("Client not found"))?
-						.base_resource_prefixes
-						.lock()
-						.clone(),
-				)
-				.ok_or_else(|| anyhow!("Resource not found"))?,
-		);
-		let _ = node.model.set(model_arc.clone());
-		Ok(model_arc)
+// No extra data needed for a simple holdout
+#[derive(Default, Asset, AsBindGroup, TypePath, Debug, Clone)]
+pub struct HoldoutExtension {}
+impl MaterialExtension for HoldoutExtension {
+	fn fragment_shader() -> ShaderRef {
+		HOLDOUT_SHADER_HANDLE.into()
 	}
 
-	fn set_material_parameter(
-		node: &Node,
-		_calling_client: Arc<Client>,
-		data: &[u8],
-	) -> Result<()> {
-		#[derive(Deserialize)]
-		struct MaterialParameterInfo {
-			idx: u32,
-			name: String,
-			value: MaterialParameter,
+	fn alpha_mode() -> Option<AlphaMode> {
+		Some(AlphaMode::Opaque)
+	}
+}
+
+#[derive(Component)]
+struct ModelNode(Weak<Model>);
+
+fn load_models(
+	asset_server: Res<AssetServer>,
+	mut cmds: Commands,
+	mut mpsc_receiver: ResMut<BevyChannelReader<(Arc<Model>, PathBuf)>>,
+) {
+	while let Some((model, path)) = mpsc_receiver.read() {
+		// idk of the asset label is the correct approach here
+		let handle = asset_server.load_with_settings(
+			GltfAssetLabel::Scene(0).from_asset(path),
+			|settings: &mut GltfLoaderSettings| {
+				settings.load_cameras = false;
+				settings.load_lights = false;
+			},
+		);
+		let entity = cmds
+			.spawn((
+				Name::new("ModelNode"),
+				SceneRoot(handle),
+				ModelNode(Arc::downgrade(&model)),
+				SpatialNode(Arc::downgrade(&model.spatial)),
+			))
+			.id();
+		model.bevy_scene_entity.set(entity.into()).unwrap();
+	}
+}
+
+fn apply_materials(
+	mut commands: Commands,
+	mut query: Query<&mut MeshMaterial3d<BevyMaterial>>,
+	mut material_registry: ResMut<MaterialRegistry>,
+	asset_server: Res<AssetServer>,
+	mut materials: ResMut<Assets<BevyMaterial>>,
+) -> bevy::prelude::Result {
+	for model_part in MODEL_REGISTRY
+		.get_valid_contents()
+		.iter()
+		.filter_map(|p| p.parts.get())
+		.flatten()
+	{
+		let entity = **model_part.mesh_entity.get().unwrap();
+		let Ok(mut mesh_mat) = query.get_mut(entity) else {
+			continue;
+		};
+		if model_part.holdout.load(Ordering::Relaxed) {
+			commands
+				.entity(entity)
+				.remove::<MeshMaterial3d<BevyMaterial>>()
+				.insert(MeshMaterial3d(HOLDOUT_MATERIAL_HANDLE));
+			continue;
 		}
-		let info: MaterialParameterInfo = deserialize(data)?;
+		if let Some(material) = model_part.pending_material_replacement.lock().take()
+			&& let Some(material) = materials.get(&material)
+		{
+			let handle = material_registry.get_handle(material.clone(), &mut materials);
+			mesh_mat.0 = handle;
+		}
+		for (param_name, param) in model_part.pending_material_parameters.lock().drain() {
+			let mut new_mat = materials.get(&mesh_mat.0).unwrap().clone();
+			param.apply_to_material(
+				&model_part.space.node().unwrap().get_client().unwrap(),
+				&mut new_mat,
+				&param_name,
+				&asset_server,
+			);
+			let handle = material_registry.get_handle(new_mat, &mut materials);
+			mesh_mat.0 = handle;
+		}
+	}
 
-		node.model
-			.get()
-			.unwrap()
-			.pending_material_parameters
+	Ok(())
+}
+
+fn gen_model_parts(
+	scenes: Res<Assets<Scene>>,
+	query: Query<(&SceneRoot, &ModelNode, &Children)>,
+	children_query: Query<&Children>,
+	part_query: Query<(&Name, Option<&Children>, &Transform), Without<Mesh3d>>,
+	part_mesh_query: Query<(&Transform, &Aabb), With<Mesh3d>>,
+	has_mesh: Query<Has<Mesh3d>>,
+	mut cmds: Commands,
+) {
+	for (scene_root, model_node, model_children) in query.iter() {
+		let Some(model) = model_node.0.upgrade() else {
+			continue;
+		};
+		if model.parts.get().is_some() {
+			continue;
+		}
+		if scenes.get(scene_root.0.id()).is_none() {
+			continue;
+		}
+		let mut parts = Vec::new();
+		for entity in model_children
+			.iter()
+			.filter_map(|e| children_query.get(e).ok())
+			.flat_map(|c| c.iter())
+		{
+			gen_path(
+				entity,
+				&part_query,
+				None,
+				&mut |entity, name, transform, parent, children| {
+					let path = parent
+						.as_ref()
+						.map(|p| format!("{}/{}", &p.path, name.as_str()))
+						.unwrap_or_else(|| name.to_string());
+					let parent_spatial = parent
+						.as_ref()
+						.map(|p| p.space.clone())
+						.unwrap_or_else(|| model.spatial.clone());
+					let client = model.spatial.node()?.get_client()?;
+					let (spatial, model_part) =
+						match model.pre_bound_parts.lock().iter().find(|v| v.path == path) {
+							None => {
+								let node =
+									client.scenegraph.add_node(Node::generate(&client, false));
+								let spatial = Spatial::add_to(
+									&node,
+									Some(parent_spatial),
+									transform.compute_matrix(),
+									false,
+								);
+								let model_part = node.add_aspect(ModelPart {
+									entity: OnceLock::new(),
+									mesh_entity: OnceLock::new(),
+									path,
+									space: spatial.clone(),
+									_model: Arc::downgrade(&model),
+									pending_material_parameters: Mutex::default(),
+									pending_material_replacement: Mutex::default(),
+									holdout: AtomicBool::new(false),
+									aliases: AliasList::default(),
+									bounds: OnceLock::new(),
+								});
+								(spatial, model_part)
+							}
+							Some(part) => {
+								part.space.set_spatial_parent(&parent_spatial).unwrap();
+								(part.space.clone(), part.clone())
+							}
+						};
+					let aabb = Aabb::enclosing(
+						children
+							.iter()
+							.flat_map(|v| v.iter())
+							.filter_map(|e| part_mesh_query.get(e).ok())
+							.flat_map(|(transform, aabb)| {
+								[
+									transform.transform_point(aabb.min().into()),
+									transform.transform_point(aabb.max().into()),
+								]
+							}),
+					)
+					.unwrap_or_default();
+					_ = spatial.bounding_box_calc.set(move |n| {
+						n.get_aspect::<ModelPart>()
+							.ok()
+							.and_then(|v| v.bounds.get().copied())
+							.unwrap_or_default()
+					});
+					spatial.set_local_transform(transform.compute_matrix());
+
+					cmds.entity(entity)
+						.insert(SpatialNode(Arc::downgrade(&spatial)));
+					let mesh_entity = children_query
+						.get(entity)
+						.iter()
+						.flat_map(|v| v.iter())
+						.find(|e| has_mesh.get(*e).unwrap_or(false))?;
+					_ = model_part.bounds.set(aabb);
+					_ = model_part.entity.set(entity.into());
+					_ = model_part.mesh_entity.set(mesh_entity.into());
+					parts.push(model_part.clone());
+					Some(model_part)
+				},
+			);
+		}
+		_ = model.parts.set(parts);
+	}
+}
+
+fn gen_path(
+	current_entity: Entity,
+	part_query: &Query<(&Name, Option<&Children>, &Transform), Without<Mesh3d>>,
+	parent: Option<Arc<ModelPart>>,
+	func: &mut dyn FnMut(
+		Entity,
+		&Name,
+		&Transform,
+		Option<Arc<ModelPart>>,
+		Option<&Children>,
+	) -> Option<Arc<ModelPart>>,
+) {
+	let Ok((name, children, transform)) = part_query.get(current_entity) else {
+		return;
+	};
+	let Some(parent) = func(current_entity, name, transform, parent, children) else {
+		return;
+	};
+	for e in children.iter().flat_map(|c| c.iter()) {
+		gen_path(e, part_query, Some(parent.clone()), func);
+	}
+}
+
+#[derive(PartialEq, Deref, DerefMut, Clone, Copy, Eq, PartialOrd, Ord, Hash)]
+struct HashedPbrMaterial(u64);
+impl HashedPbrMaterial {
+	fn new(material: &BevyMaterial) -> Self {
+		let mut hasher = FxHasher::default();
+		Self::hash_pbr_mat(material, &mut hasher);
+		Self(hasher.finish())
+	}
+	fn hash_pbr_mat<H: Hasher>(mat: &BevyMaterial, state: &mut H) {
+		hash_color(mat.base_color, state);
+		hash_color(mat.emissive.into(), state);
+		state.write_u32(mat.metallic.to_bits());
+		state.write_u32(mat.perceptual_roughness.to_bits());
+		match mat.alpha_mode {
+			AlphaMode::Opaque => state.write_u8(0),
+			AlphaMode::Mask(v) => {
+				state.write_u8(1);
+				state.write_u32(v.to_bits());
+			}
+			AlphaMode::Blend => state.write_u8(2),
+			AlphaMode::Premultiplied => state.write_u8(3),
+			AlphaMode::AlphaToCoverage => state.write_u8(4),
+			AlphaMode::Add => state.write_u8(5),
+			AlphaMode::Multiply => state.write_u8(6),
+		}
+		state.write_u8(mat.double_sided as u8);
+		mat.base_color_texture.hash(state);
+		mat.emissive_texture.hash(state);
+		mat.metallic_roughness_texture.hash(state);
+		mat.occlusion_texture.hash(state);
+		// should always be the same, TODO: make the spherical harmonics buffer a per mesh instance thing
+		// mat.spherical_harmonics.hash(state);
+	}
+}
+fn hash_color<H: Hasher>(color: Color, state: &mut H) {
+	match color {
+		Color::Srgba(srgba) => {
+			state.write_u8(0);
+			state.write(&srgba.to_u8_array());
+		}
+		Color::LinearRgba(linear_rgba) => {
+			state.write_u8(1);
+			state.write(&linear_rgba.to_u8_array());
+		}
+		Color::Hsla(hsla) => {
+			state.write_u8(2);
+			hsla.to_f32_array()
+				.iter()
+				.for_each(|v| state.write_u32(v.to_bits()));
+		}
+		Color::Hsva(hsva) => {
+			state.write_u8(3);
+			hsva.to_f32_array()
+				.iter()
+				.for_each(|v| state.write_u32(v.to_bits()));
+		}
+		Color::Hwba(hwba) => {
+			state.write_u8(4);
+			hwba.to_f32_array()
+				.iter()
+				.for_each(|v| state.write_u32(v.to_bits()));
+		}
+		Color::Laba(laba) => {
+			state.write_u8(5);
+			laba.to_f32_array()
+				.iter()
+				.for_each(|v| state.write_u32(v.to_bits()));
+		}
+		Color::Lcha(lcha) => {
+			state.write_u8(6);
+			lcha.to_f32_array()
+				.iter()
+				.for_each(|v| state.write_u32(v.to_bits()));
+		}
+		Color::Oklaba(oklaba) => {
+			state.write_u8(7);
+			oklaba
+				.to_f32_array()
+				.iter()
+				.for_each(|v| state.write_u32(v.to_bits()));
+		}
+		Color::Oklcha(oklcha) => {
+			state.write_u8(8);
+			oklcha
+				.to_f32_array()
+				.iter()
+				.for_each(|v| state.write_u32(v.to_bits()));
+		}
+		Color::Xyza(xyza) => {
+			state.write_u8(9);
+			xyza.to_f32_array()
+				.iter()
+				.for_each(|v| state.write_u32(v.to_bits()));
+		}
+	}
+}
+static MODEL_REGISTRY: Registry<Model> = Registry::new();
+
+impl MaterialParameter {
+	fn apply_to_material(
+		&self,
+		client: &Client,
+		mat: &mut BevyMaterial,
+		parameter_name: &str,
+		asset_server: &AssetServer,
+	) {
+		match self {
+			MaterialParameter::Bool(val) => match parameter_name {
+				"double_sided" => mat.double_sided = *val,
+				v => {
+					error!("unknown param_name ({v}) for color")
+				}
+			},
+			MaterialParameter::Int(_val) => {
+				// nothing uses an int
+			}
+			MaterialParameter::UInt(_val) => {
+				// nothing uses an uint
+			}
+			MaterialParameter::Float(val) => {
+				match parameter_name {
+					"metallic" => mat.metallic = *val,
+					"roughness" => mat.perceptual_roughness = *val,
+					// we probably don't want to expose tex_scale
+					// "tex_scale" => mat.tex_scale = *val,
+					v => {
+						error!("unknown param_name ({v}) for float")
+					}
+				}
+			}
+			MaterialParameter::Vec2(_val) => {
+				// nothing uses a Vec2
+			}
+			MaterialParameter::Vec3(_val) => {
+				// nothing uses a Vec3
+			}
+			MaterialParameter::Color(color) => match parameter_name {
+				"color" => mat.base_color = color.to_bevy(),
+				"emission_factor" => mat.emissive = color.to_bevy().to_linear(),
+				v => {
+					error!("unknown param_name ({v}) for color")
+				}
+			},
+			MaterialParameter::Texture(resource) => {
+				let Some(texture_path) =
+					get_resource_file(resource, client, &[OsStr::new("png"), OsStr::new("jpg")])
+				else {
+					return;
+				};
+				let handle = asset_server.load(texture_path);
+				match parameter_name {
+					"diffuse" => mat.base_color_texture = Some(handle),
+					"emission" => mat.emissive_texture = Some(handle),
+					"metal" => mat.metallic_roughness_texture = Some(handle),
+					"occlusion" => mat.occlusion_texture = Some(handle),
+					v => {
+						error!("unknown param_name ({v}) for texture");
+					}
+				}
+				// mat.alpha_mode = AlphaMode::Blend;
+			}
+		}
+	}
+}
+
+pub struct ModelPart {
+	entity: OnceLock<EntityHandle>,
+	mesh_entity: OnceLock<EntityHandle>,
+	path: String,
+	space: Arc<Spatial>,
+	_model: Weak<Model>,
+	pending_material_parameters: Mutex<FxHashMap<String, MaterialParameter>>,
+	pending_material_replacement: Mutex<Option<Handle<BevyMaterial>>>,
+	holdout: AtomicBool,
+	aliases: AliasList,
+	bounds: OnceLock<Aabb>,
+}
+impl ModelPart {
+	pub fn replace_material(&self, replacement: Handle<BevyMaterial>) {
+		self.pending_material_replacement
 			.lock()
-			.insert((info.idx, info.name), info.value);
-
+			.replace(replacement);
+	}
+	pub fn set_material_parameter(&self, parameter_name: String, value: MaterialParameter) {
+		self.pending_material_parameters
+			.lock()
+			.insert(parameter_name, value);
+	}
+}
+impl ModelPartAspect for ModelPart {
+	#[doc = "Set this model part's material to one that cuts a hole in the world. Often used for overlays/passthrough where you want to show the background through an object."]
+	fn apply_holdout_material(node: Arc<Node>, _calling_client: Arc<Client>) -> Result<()> {
+		let model_part = node.get_aspect::<ModelPart>()?;
+		model_part.holdout.store(true, Ordering::Relaxed);
 		Ok(())
 	}
 
-	fn draw(&self, sk: &StereoKit, draw_ctx: &DrawContext) {
-		let sk_model = self
-			.sk_model
-			.get_or_try_init(|| {
-				self.pending_model_path
-					.get()
-					.and_then(|path| SKModel::from_file(sk, path.as_path(), None))
-					.as_ref()
-					.cloned()
-					.map(SendWrapper::new)
-					.ok_or(Error)
-			})
-			.ok();
-
-		if let Some(sk_model) = sk_model {
-			{
-				let mut material_replacements = self.pending_material_replacements.lock();
-				for (material_idx, replacement_material) in material_replacements.iter() {
-					sk_model.set_material(*material_idx as i32, replacement_material);
-				}
-				material_replacements.clear();
+	#[doc = "Set the material parameter with `parameter_name` to `value`"]
+	fn set_material_parameter(
+		node: Arc<Node>,
+		_calling_client: Arc<Client>,
+		parameter_name: String,
+		value: MaterialParameter,
+	) -> Result<()> {
+		let model_part = node.get_aspect::<ModelPart>()?;
+		model_part.set_material_parameter(parameter_name, value);
+		Ok(())
+	}
+}
+#[derive(Default, Resource)]
+pub struct MaterialRegistry(FxHashMap<HashedPbrMaterial, Handle<BevyMaterial>>);
+impl MaterialRegistry {
+	/// returns strong handle for PbrMaterial elminitating duplications
+	pub fn get_handle(
+		&mut self,
+		material: BevyMaterial,
+		materials: &mut ResMut<Assets<BevyMaterial>>,
+	) -> Handle<BevyMaterial> {
+		let hash = HashedPbrMaterial::new(&material);
+		match self
+			.0
+			.get(&hash)
+			.and_then(|v| materials.get_strong_handle(v.id()))
+		{
+			Some(v) => v,
+			None => {
+				let handle = materials.add(material);
+				self.0.insert(hash, handle.clone_weak());
+				handle
 			}
-
-			{
-				let mut material_parameters = self.pending_material_parameters.lock();
-				for ((material_idx, parameter_name), parameter_value) in material_parameters.iter()
-				{
-					if let Some(material) = sk_model.get_material(sk, *material_idx as i32) {
-						match parameter_value {
-							MaterialParameter::Texture(path) => {
-								if let Some(tex) = Texture::from_file(sk, path.as_path(), true, 0) {
-									material.set_parameter(parameter_name.as_str(), &tex);
-								}
-							}
-						}
-					}
-				}
-				material_parameters.clear();
-			}
-
-			let global_transform = self.space.global_transform().into();
-			sk_model.draw(
-				draw_ctx,
-				global_transform,
-				Rgba::new(Rgb::new(1_f32, 1_f32, 1_f32), 1_f32),
-				RenderLayer::Layer0,
-			);
 		}
 	}
 }
+
+pub struct Model {
+	spatial: Arc<Spatial>,
+	_resource_id: ResourceID,
+	bevy_scene_entity: OnceLock<EntityHandle>,
+	parts: OnceLock<Vec<Arc<ModelPart>>>,
+	pre_bound_parts: Mutex<Vec<Arc<ModelPart>>>,
+}
+impl Model {
+	pub fn add_to(node: &Arc<Node>, resource_id: ResourceID) -> Result<Arc<Model>> {
+		let pending_model_path = get_resource_file(
+			&resource_id,
+			&*node.get_client().ok_or_else(|| eyre!("Client not found"))?,
+			&[OsStr::new("glb"), OsStr::new("gltf")],
+		)
+		.ok_or_else(|| eyre!("Resource not found"))?;
+
+		let model = Arc::new(Model {
+			spatial: node.get_aspect::<Spatial>().unwrap().clone(),
+			_resource_id: resource_id,
+			bevy_scene_entity: OnceLock::new(),
+			pre_bound_parts: Mutex::default(),
+			parts: OnceLock::new(),
+		});
+		LOAD_MODEL
+			.send((model.clone(), pending_model_path))
+			.unwrap();
+		MODEL_REGISTRY.add_raw(&model);
+
+		node.add_aspect_raw(model.clone());
+		Ok(model)
+	}
+	pub fn get_model_part(self: &Arc<Self>, part_path: String) -> Result<Arc<ModelPart>> {
+		let part = match self
+			.parts
+			.get()
+			.map(|v| v.iter().find(|p| p.path == part_path))
+		{
+			Some(Some(part)) => part.clone(),
+			Some(None) => {
+				let paths = self
+					.parts
+					.get()
+					.unwrap()
+					.iter()
+					.map(|p| &p.path)
+					.collect::<Vec<_>>();
+				bail!(
+					"Couldn't find model part at path {part_path}, all available paths: {paths:?}",
+				);
+			}
+			None => {
+				// TODO: this could be a denail of service vector
+				let client = self.spatial.node().unwrap().get_client().unwrap();
+				let part_node = client.scenegraph.add_node(Node::generate(&client, false));
+				let spatial = Spatial::add_to(
+					&part_node,
+					Some(self.spatial.clone()),
+					Mat4::IDENTITY,
+					false,
+				);
+				let part = part_node.add_aspect(ModelPart {
+					entity: OnceLock::new(),
+					mesh_entity: OnceLock::new(),
+					path: part_path,
+					space: spatial,
+					_model: Arc::downgrade(self),
+					pending_material_parameters: Mutex::default(),
+					pending_material_replacement: Mutex::default(),
+					holdout: AtomicBool::new(false),
+					aliases: AliasList::default(),
+					bounds: OnceLock::new(),
+				});
+				self.pre_bound_parts.lock().push(part.clone());
+				part
+			}
+		};
+		Ok(part)
+	}
+}
+impl ModelAspect for Model {
+	#[doc = "Bind a model part to the node with the ID input."]
+	fn bind_model_part(
+		node: Arc<Node>,
+		calling_client: Arc<Client>,
+		id: u64,
+		part_path: String,
+	) -> Result<()> {
+		let model = node.get_aspect::<Model>()?;
+		let part = model.get_model_part(part_path)?;
+		Alias::create_with_id(
+			&part.space.node().unwrap(),
+			&calling_client,
+			id,
+			MODEL_PART_ASPECT_ALIAS_INFO.clone(),
+			Some(&part.aliases),
+		)?;
+		Ok(())
+	}
+}
 impl Drop for Model {
 	fn drop(&mut self) {
-		if let Some(model) = self.sk_model.take() {
-			destroy_queue::add(model);
-		}
 		MODEL_REGISTRY.remove(self);
 	}
 }
-
-pub fn draw_all(sk: &StereoKit, draw_ctx: &DrawContext) {
-	for model in MODEL_REGISTRY.get_valid_contents() {
-		model.draw(sk, draw_ctx);
-	}
-}
-
-pub fn create_flex(_node: &Node, calling_client: Arc<Client>, data: &[u8]) -> Result<()> {
-	#[derive(Deserialize)]
-	struct CreateModelInfo<'a> {
-		name: &'a str,
-		parent_path: &'a str,
-		transform: Transform,
-		resource: ResourceID,
-	}
-	let info: CreateModelInfo = deserialize(data)?;
-	let node = Node::create(&calling_client, "/drawable/model", info.name, true);
-	let parent = find_spatial_parent(&calling_client, info.parent_path)?;
-	let transform = parse_transform(info.transform, true, true, true)?;
-	let node = node.add_to_scenegraph();
-	Spatial::add_to(&node, Some(parent), transform)?;
-	Model::add_to(&node, info.resource)?;
-	Ok(())
-}

src/nodes/drawable/text.rs

@@ -1,211 +1,241 @@
 use crate::{
-	core::{client::Client, destroy_queue, registry::Registry, resource::ResourceID},
+	BevyMaterial,
+	core::{
+		bevy_channel::{BevyChannel, BevyChannelReader},
+		client::Client,
+		color::ColorConvert,
+		entity_handle::EntityHandle,
+		error::Result,
+		registry::Registry,
+		resource::get_resource_file,
+	},
 	nodes::{
-		spatial::{find_spatial_parent, parse_transform, Spatial},
 		Node,
+		drawable::XAlign,
+		spatial::{Spatial, SpatialNode},
 	},
 };
-use anyhow::{anyhow, ensure, Result};
-use glam::{vec3, Mat4, Vec2};
-use mint::Vector2;
-use once_cell::sync::OnceCell;
-use parking_lot::Mutex;
-use prisma::{Flatten, Rgb, Rgba};
-use send_wrapper::SendWrapper;
-use serde::Deserialize;
-use stardust_xr::{schemas::flex::deserialize, values::Transform};
-use std::{path::PathBuf, sync::Arc};
-use stereokit::{
-	font::Font,
-	lifecycle::DrawContext,
-	text::{self, TextAlign, TextFit, TextStyle},
-	StereoKit,
+use bevy::{platform::collections::HashMap, prelude::*, render::mesh::MeshAabb};
+use bevy_mesh_text_3d::{
+	Align, Attrs, MeshTextPlugin, Settings as FontSettings, generate_meshes,
+	text_glyphs::TextGlyphs,
 };
+use color_eyre::eyre::eyre;
+use core::f32;
+use cosmic_text::Metrics;
+use parking_lot::Mutex;
+use std::{ffi::OsStr, mem, path::PathBuf, sync::Arc};
+
+static SPAWN_TEXT: BevyChannel<Arc<Text>> = BevyChannel::new();
+
+pub struct TextNodePlugin;
+
+impl Plugin for TextNodePlugin {
+	fn build(&self, app: &mut App) {
+		// Text init stuff
+		// 1.0 for font size in meters
+		app.add_plugins(MeshTextPlugin::new(1.0));
+		app.world_mut()
+			.resource_mut::<FontSettings>()
+			.font_system
+			.db_mut()
+			.load_system_fonts();
+
+		SPAWN_TEXT.init(app);
+		app.init_resource::<MaterialRegistry>();
+		app.add_systems(Update, spawn_text);
+	}
+}
+
+fn spawn_text(
+	mut mpsc: ResMut<BevyChannelReader<Arc<Text>>>,
+	mut cmds: Commands,
+	mut font_settings: ResMut<FontSettings>,
+	mut material_registry: ResMut<MaterialRegistry>,
+	mut materials: ResMut<Assets<BevyMaterial>>,
+	mut meshes: ResMut<Assets<Mesh>>,
+	mut font_registry: Local<FontDatabaseRegistry>,
+) {
+	while let Some(text) = mpsc.read() {
+		if let Some(entity) = text.entity.lock().take() {
+			cmds.entity(*entity).despawn();
+		}
+		let style = text.data.lock();
+		let old_db = text.font_path.clone().map(|p| {
+			let db = font_registry.get(p);
+			mem::swap(font_settings.font_system.db_mut(), db);
+			db
+		});
+		let attrs = Attrs::new().weight(cosmic_text::Weight::BOLD);
+		let alignment = Some(match style.text_align_x {
+			super::XAlign::Left => Align::Right,
+			super::XAlign::Center => Align::Center,
+			super::XAlign::Right => Align::Left,
+		});
+		let text_string = text.text.lock().clone();
+		let mut text_glyphs = TextGlyphs::new(
+			Metrics {
+				font_size: style.character_height,
+				line_height: style.character_height,
+			},
+			[(text_string.as_str(), attrs.clone())],
+			&attrs,
+			&mut font_settings.font_system,
+			alignment,
+		);
+		let max_width = style.bounds.as_ref().map(|v| v.bounds.x);
+		let max_height = style.bounds.as_ref().map(|v| v.bounds.x);
+		let (width, _height) =
+			text_glyphs.measure(max_width, max_height, &mut font_settings.font_system);
+		let char_meshes = generate_meshes(
+			bevy_mesh_text_3d::InputText::Simple {
+				text: text_string,
+				material: material_registry.get_handle(
+					BevyMaterial {
+						base_color: style.color.to_bevy(),
+						emissive: Color::WHITE.to_linear(),
+						metallic: 0.0,
+						perceptual_roughness: 1.0,
+						// If alpha is supported on text we need to change this
+						alpha_mode: AlphaMode::Opaque,
+						double_sided: false,
+						..default()
+					},
+					&mut materials,
+				),
+				attrs,
+			},
+			&mut font_settings,
+			bevy_mesh_text_3d::Parameters {
+				extrusion_depth: 0.0,
+				font_size: style.character_height,
+				line_height: style.character_height,
+				alignment,
+				max_width,
+				max_height,
+			},
+			&mut meshes,
+		);
+		if let Some(db) = old_db {
+			mem::swap(font_settings.font_system.db_mut(), db);
+		}
+		let Ok(char_meshes) =
+			char_meshes.inspect_err(|err| error!("unable to create text meshes: {err}"))
+		else {
+			continue;
+		};
+		let dist = char_meshes.iter().fold(f32::MAX, |dist, v| {
+			dist.min(
+				v.transform.translation.x
+					- meshes
+						.get(&v.mesh)
+						.unwrap()
+						.compute_aabb()
+						.unwrap_or_default()
+						.half_extents
+						.x,
+			)
+		});
+		// TODO: text align
+		let letters = char_meshes
+			.into_iter()
+			.map(|v| {
+				cmds.spawn((
+					Mesh3d(v.mesh),
+					MeshMaterial3d(v.material),
+					// rotation is sus, might be related to the gltf coordinate system
+					Transform::from_rotation(Quat::from_rotation_y(f32::consts::PI))
+						* Transform::from_xyz(
+							-dist
+								+ match style.bounds.as_ref().map(|v| v.anchor_align_x) {
+									Some(XAlign::Center) => width * -0.5,
+									Some(XAlign::Right) => -width,
+									Some(XAlign::Left) => 0.0,
+									None => 0.0,
+								},
+							0.0,
+							0.0,
+						) * v.transform,
+				))
+				.id()
+			})
+			.collect::<Vec<_>>();
+		let entity = cmds
+			.spawn((
+				Name::new("TextNode"),
+				SpatialNode(Arc::downgrade(&text.spatial)),
+			))
+			.add_children(&letters)
+			.id();
+		text.entity.lock().replace(EntityHandle(entity));
+	}
+}
+
+#[derive(Default)]
+struct FontDatabaseRegistry(HashMap<PathBuf, cosmic_text::fontdb::Database>);
+impl FontDatabaseRegistry {
+	fn get(&mut self, path: PathBuf) -> &mut cosmic_text::fontdb::Database {
+		self.0.entry(path).or_insert_with_key(|path| {
+			let mut db = cosmic_text::fontdb::Database::new();
+			if let Err(err) = db.load_font_file(path) {
+				error!("unable to load font file {} {err}", path.to_string_lossy());
+			};
+			db
+		})
+	}
+}
+
+use super::{TextAspect, TextStyle, model::MaterialRegistry};
 
 static TEXT_REGISTRY: Registry<Text> = Registry::new();
 
-struct TextData {
-	text: String,
-	character_height: f32,
-	text_align: TextAlign,
-	bounds: Option<Vec2>,
-	fit: TextFit,
-	bounds_align: TextAlign,
-	color: Rgba<f32>,
-}
-
 pub struct Text {
-	space: Arc<Spatial>,
+	spatial: Arc<Spatial>,
 	font_path: Option<PathBuf>,
-	style: OnceCell<SendWrapper<TextStyle>>,
-
-	data: Mutex<TextData>,
+	entity: Mutex<Option<EntityHandle>>,
+	text: Mutex<String>,
+	data: Mutex<TextStyle>,
 }
 impl Text {
-	#[allow(clippy::too_many_arguments)]
-	pub fn add_to(
-		node: &Arc<Node>,
-		font_resource_id: Option<ResourceID>,
-		text: String,
-		character_height: f32,
-		text_align: TextAlign,
-		bounds: Option<Vector2<f32>>,
-		fit: TextFit,
-		bounds_align: TextAlign,
-		color: Rgba<f32>,
-	) -> Result<Arc<Text>> {
-		ensure!(
-			node.spatial.get().is_some(),
-			"Internal: Node does not have a spatial attached!"
-		);
-		ensure!(
-			node.model.get().is_none(),
-			"Internal: Node already has text attached!"
-		);
-
-		let client = node
-			.get_client()
-			.ok_or_else(|| anyhow!("Client not found"))?;
+	pub fn add_to(node: &Arc<Node>, text: String, style: TextStyle) -> Result<Arc<Text>> {
+		let client = node.get_client().ok_or_else(|| eyre!("Client not found"))?;
 		let text = TEXT_REGISTRY.add(Text {
-			space: node.spatial.get().unwrap().clone(),
-			font_path: font_resource_id
-				.and_then(|res| res.get_file(&client.base_resource_prefixes.lock().clone())),
-			style: OnceCell::new(),
-
-			data: Mutex::new(TextData {
-				text,
-				character_height,
-				text_align,
-				bounds: bounds.map(|b| b.into()),
-				fit,
-				bounds_align,
-				color,
+			spatial: node.get_aspect::<Spatial>().unwrap().clone(),
+			font_path: style.font.as_ref().and_then(|res| {
+				get_resource_file(res, &client, &[OsStr::new("ttf"), OsStr::new("otf")])
 			}),
+
+			entity: Mutex::new(None),
+			text: Mutex::new(text),
+			data: Mutex::new(style),
 		});
-		node.add_local_signal("setCharacterHeight", Text::set_character_height_flex);
-		node.add_local_signal("setText", Text::set_text_flex);
-		let _ = node.text.set(text.clone());
+		node.add_aspect_raw(text.clone());
+		_ = SPAWN_TEXT.send(text.clone());
 
 		Ok(text)
 	}
-
-	fn draw(&self, sk: &StereoKit, draw_ctx: &DrawContext) {
-		let style =
-			self.style
-				.get_or_try_init(|| -> Result<SendWrapper<TextStyle>, anyhow::Error> {
-					let font = if let Some(path) = self.font_path.as_deref() {
-						Font::from_file(sk, path)
-					} else {
-						Some(Font::default(sk))
-					};
-					Ok(SendWrapper::new(TextStyle::new(
-						sk,
-						font.ok_or(std::fmt::Error)?,
-						1.0,
-						Rgba::new(Rgb::new(1.0, 1.0, 1.0), 1.0),
-					)))
-				});
-
-		if let Ok(style) = style {
-			let data = self.data.lock();
-			let transform = self.space.global_transform()
-				* Mat4::from_scale(vec3(
-					data.character_height,
-					data.character_height,
-					data.character_height,
-				));
-			if let Some(bounds) = data.bounds {
-				text::draw_in(
-					draw_ctx,
-					&data.text,
-					transform,
-					bounds / data.character_height,
-					data.fit,
-					style,
-					data.bounds_align,
-					data.text_align,
-					vec3(0.0, 0.0, 0.0),
-					data.color,
-				);
-			} else {
-				text::draw_at(
-					draw_ctx,
-					&data.text,
-					transform,
-					style,
-					data.bounds_align,
-					data.text_align,
-					vec3(0.0, 0.0, 0.0),
-					data.color,
-				);
-			}
-		}
-	}
-
-	pub fn set_character_height_flex(
-		node: &Node,
+}
+impl TextAspect for Text {
+	fn set_character_height(
+		node: Arc<Node>,
 		_calling_client: Arc<Client>,
-		data: &[u8],
+		height: f32,
 	) -> Result<()> {
-		let height = flexbuffers::Reader::get_root(data)?.get_f64()? as f32;
-		node.text.get().unwrap().data.lock().character_height = height;
+		let this_text = node.get_aspect::<Text>()?;
+		this_text.data.lock().character_height = height;
+		_ = SPAWN_TEXT.send(this_text);
 		Ok(())
 	}
 
-	pub fn set_text_flex(node: &Node, _calling_client: Arc<Client>, data: &[u8]) -> Result<()> {
-		let text = flexbuffers::Reader::get_root(data)?.get_str()?.to_string();
-		node.text.get().unwrap().data.lock().text = text;
+	fn set_text(node: Arc<Node>, _calling_client: Arc<Client>, text: String) -> Result<()> {
+		let this_text = node.get_aspect::<Text>()?;
+		*this_text.text.lock() = text;
+		_ = SPAWN_TEXT.send(this_text);
 		Ok(())
 	}
 }
 impl Drop for Text {
 	fn drop(&mut self) {
-		if let Some(style) = self.style.take() {
-			destroy_queue::add(style);
-		}
 		TEXT_REGISTRY.remove(self);
 	}
 }
-
-pub fn draw_all(sk: &StereoKit, draw_ctx: &DrawContext) {
-	for text in TEXT_REGISTRY.get_valid_contents() {
-		text.draw(sk, draw_ctx);
-	}
-}
-
-pub fn create_flex(_node: &Node, calling_client: Arc<Client>, data: &[u8]) -> Result<()> {
-	#[derive(Deserialize)]
-	struct CreateTextInfo<'a> {
-		name: &'a str,
-		parent_path: &'a str,
-		transform: Transform,
-		text: String,
-		font_resource: Option<ResourceID>,
-		character_height: f32,
-		text_align: TextAlign,
-		bounds: Option<Vector2<f32>>,
-		fit: TextFit,
-		bounds_align: TextAlign,
-		color: [f32; 4],
-	}
-	let info: CreateTextInfo = deserialize(data)?;
-	let node = Node::create(&calling_client, "/drawable/text", info.name, true);
-	let parent = find_spatial_parent(&calling_client, info.parent_path)?;
-	let transform = parse_transform(info.transform, true, true, true)?;
-	let color = Rgba::from_slice(&info.color);
-
-	let node = node.add_to_scenegraph();
-	Spatial::add_to(&node, Some(parent), transform)?;
-	Text::add_to(
-		&node,
-		info.font_resource,
-		info.text,
-		info.character_height,
-		info.text_align,
-		info.bounds,
-		info.fit,
-		info.bounds_align,
-		color,
-	)?;
-	Ok(())
-}

src/nodes/fields.rs

@@ -0,0 +1,402 @@
+use super::alias::{Alias, AliasInfo};
+use super::spatial::{
+	SPATIAL_REF_GET_LOCAL_BOUNDING_BOX_SERVER_OPCODE,
+	SPATIAL_REF_GET_RELATIVE_BOUNDING_BOX_SERVER_OPCODE, SPATIAL_REF_GET_TRANSFORM_SERVER_OPCODE,
+	Spatial,
+};
+use super::{Aspect, AspectIdentifier, Node};
+use crate::DbusConnection;
+use crate::core::client::Client;
+use crate::core::error::Result;
+use crate::core::registry::Registry;
+use crate::nodes::spatial::SPATIAL_ASPECT_ALIAS_INFO;
+use crate::nodes::spatial::SPATIAL_REF_ASPECT_ALIAS_INFO;
+use crate::nodes::spatial::Transform;
+use bevy::app::{Plugin, Update};
+use bevy::color::Color;
+use bevy::ecs::resource::Resource;
+use bevy::ecs::schedule::IntoScheduleConfigs;
+use bevy::ecs::system::Res;
+use bevy::gizmos::gizmos::Gizmos;
+use bevy::gizmos::primitives::dim3::GizmoPrimitive3d;
+use bevy::math::primitives::{Cylinder, Torus};
+use color_eyre::eyre::OptionExt;
+use glam::{Vec3, Vec3A, Vec3Swizzles, vec2, vec3, vec3a};
+use parking_lot::Mutex;
+use rustc_hash::FxHashMap;
+use stardust_xr::values::Vector3;
+use std::sync::{Arc, LazyLock, Weak};
+use zbus::interface;
+
+// TODO: get SDFs working properly with non-uniform scale and so on, output distance relative to the spatial it's compared against
+
+pub static FIELD_ALIAS_INFO: LazyLock<AliasInfo> = LazyLock::new(|| AliasInfo {
+	server_methods: vec![
+		SPATIAL_REF_GET_TRANSFORM_SERVER_OPCODE,
+		SPATIAL_REF_GET_LOCAL_BOUNDING_BOX_SERVER_OPCODE,
+		SPATIAL_REF_GET_RELATIVE_BOUNDING_BOX_SERVER_OPCODE,
+		FIELD_REF_DISTANCE_SERVER_OPCODE,
+		FIELD_REF_NORMAL_SERVER_OPCODE,
+		FIELD_REF_CLOSEST_POINT_SERVER_OPCODE,
+		FIELD_REF_RAY_MARCH_SERVER_OPCODE,
+	],
+	..Default::default()
+});
+
+pub struct FieldDebugGizmoPlugin;
+impl Plugin for FieldDebugGizmoPlugin {
+	fn build(&self, app: &mut bevy::app::App) {
+		let (tx, rx) = tokio::sync::watch::channel(false);
+		let conn = app.world().resource::<DbusConnection>().0.clone();
+		tokio::spawn(async move {
+			_ = conn
+				.object_server()
+				.at("/org/stardustxr/Server", FieldDebugGizmos { state: tx })
+				.await;
+		});
+		app.insert_resource(FieldDebugGizmosEnabled(rx));
+		app.add_systems(
+			Update,
+			draw_field_gizmos.run_if(|res: Res<FieldDebugGizmosEnabled>| *res.0.borrow()),
+		);
+	}
+}
+
+#[derive(Resource)]
+struct FieldDebugGizmosEnabled(tokio::sync::watch::Receiver<bool>);
+
+fn draw_field_gizmos(mut gizmos: Gizmos) {
+	FIELD_REGISTRY_DEBUG_GIZMOS
+		.get_valid_contents()
+		.iter()
+		.for_each(|f| {
+			let transform =
+				bevy::transform::components::Transform::from_matrix(f.spatial.global_transform());
+			let color = Color::srgb_u8(0x04, 0xFD, 0x4C);
+			match f.shape.lock().clone() {
+				Shape::Box(size) => gizmos.cuboid(transform.with_scale(size.into()), color),
+				Shape::Cylinder(CylinderShape { length, radius }) => {
+					gizmos
+						.primitive_3d(
+							&Cylinder {
+								radius,
+								half_height: length * 0.5,
+							},
+							transform.to_isometry(),
+							color,
+						)
+						.resolution(32);
+				}
+				Shape::Sphere(radius) => {
+					gizmos.sphere(transform.to_isometry(), radius, color);
+				}
+				Shape::Torus(TorusShape { radius_a, radius_b }) => {
+					let minor_radius;
+					let major_radius;
+					if radius_a >= radius_b {
+						major_radius = radius_a;
+						minor_radius = radius_b;
+					} else {
+						major_radius = radius_b;
+						minor_radius = radius_a;
+					}
+					gizmos
+						.primitive_3d(
+							&Torus {
+								minor_radius,
+								major_radius,
+							},
+							transform.to_isometry(),
+							color,
+						)
+						.minor_resolution(32)
+						.major_resolution(32);
+				}
+			}
+		});
+}
+
+struct FieldDebugGizmos {
+	state: tokio::sync::watch::Sender<bool>,
+}
+
+#[interface(name = "org.stardustxr.debug.FieldDebugGizmos")]
+impl FieldDebugGizmos {
+	fn enable(&mut self) {
+		_ = self.state.send(true);
+	}
+	fn disable(&mut self) {
+		_ = self.state.send(false);
+	}
+}
+
+static FIELD_REGISTRY_DEBUG_GIZMOS: Registry<Field> = Registry::new();
+
+stardust_xr_server_codegen::codegen_field_protocol!();
+
+lazy_static::lazy_static! {
+	pub static ref EXPORTED_FIELDS: Mutex<FxHashMap<u64, Weak<Node>>> = Mutex::new(FxHashMap::default());
+}
+
+pub trait FieldTrait: Send + Sync + 'static {
+	fn spatial_ref(&self) -> &Spatial;
+
+	fn local_distance(&self, p: Vec3A) -> f32;
+	fn local_normal(&self, p: Vec3A, r: f32) -> Vec3A {
+		let d = self.local_distance(p);
+		let e = vec2(r, 0_f32);
+
+		let n = vec3a(d, d, d)
+			- vec3a(
+				self.local_distance(vec3a(e.x, e.y, e.y)),
+				self.local_distance(vec3a(e.y, e.x, e.y)),
+				self.local_distance(vec3a(e.y, e.y, e.x)),
+			);
+
+		n.normalize()
+	}
+	fn local_closest_point(&self, p: Vec3A, r: f32) -> Vec3A {
+		p - (self.local_normal(p, r) * self.local_distance(p))
+	}
+
+	fn distance(&self, reference_space: &Spatial, p: Vec3A) -> f32 {
+		let reference_to_local_space =
+			Spatial::space_to_space_matrix(Some(reference_space), Some(self.spatial_ref()));
+		let local_p = reference_to_local_space.transform_point3a(p);
+		self.local_distance(local_p)
+	}
+	fn normal(&self, reference_space: &Spatial, p: Vec3A, r: f32) -> Vec3A {
+		let reference_to_local_space =
+			Spatial::space_to_space_matrix(Some(reference_space), Some(self.spatial_ref()));
+		let local_p = reference_to_local_space.transform_point3a(p);
+		reference_to_local_space
+			.inverse()
+			.transform_vector3a(self.local_normal(local_p, r))
+	}
+	fn closest_point(&self, reference_space: &Spatial, p: Vec3A, r: f32) -> Vec3A {
+		let reference_to_local_space =
+			Spatial::space_to_space_matrix(Some(reference_space), Some(self.spatial_ref()));
+		let local_p = reference_to_local_space.transform_point3a(p);
+		reference_to_local_space
+			.inverse()
+			.transform_point3a(self.local_closest_point(local_p, r))
+	}
+
+	fn ray_march(&self, ray: Ray) -> RayMarchResult {
+		let mut result = RayMarchResult {
+			ray_origin: ray.origin.into(),
+			ray_direction: ray.direction.into(),
+			min_distance: f32::MAX,
+			deepest_point_distance: 0_f32,
+			ray_length: 0_f32,
+			ray_steps: 0,
+		};
+
+		let ray_to_field_matrix =
+			Spatial::space_to_space_matrix(Some(&ray.space), Some(self.spatial_ref()));
+		let mut ray_point = ray_to_field_matrix.transform_point3a(ray.origin.into());
+		let ray_direction = ray_to_field_matrix
+			.transform_vector3a(ray.direction.into())
+			.normalize();
+
+		while result.ray_steps < MAX_RAY_STEPS && result.ray_length < MAX_RAY_LENGTH {
+			let distance = self.local_distance(ray_point);
+			let march_distance = distance.clamp(MIN_RAY_MARCH, MAX_RAY_MARCH);
+
+			result.ray_length += march_distance;
+			ray_point += ray_direction * march_distance;
+
+			if result.min_distance > distance {
+				result.deepest_point_distance = result.ray_length;
+				result.min_distance = distance;
+			}
+
+			result.ray_steps += 1;
+		}
+
+		result
+	}
+}
+
+pub struct Ray {
+	pub origin: Vec3,
+	pub direction: Vec3,
+	pub space: Arc<Spatial>,
+}
+
+// const MIN_RAY_STEPS: u32 = 0;
+const MAX_RAY_STEPS: u32 = 1000;
+
+const MIN_RAY_MARCH: f32 = 0.001_f32;
+const MAX_RAY_MARCH: f32 = f32::MAX;
+
+// const MIN_RAY_LENGTH: f32 = 0_f32;
+const MAX_RAY_LENGTH: f32 = 1000_f32;
+
+pub struct Field {
+	pub spatial: Arc<Spatial>,
+	pub shape: Mutex<Shape>,
+}
+impl Field {
+	pub fn add_to(node: &Arc<Node>, shape: Shape) -> Result<Arc<Field>> {
+		let spatial = node.get_aspect::<Spatial>()?;
+		let field = Field {
+			spatial,
+			shape: Mutex::new(shape),
+		};
+		let field = node.add_aspect(field);
+		FIELD_REGISTRY_DEBUG_GIZMOS.add_raw(&field);
+		node.add_aspect(FieldRef);
+		Ok(field)
+	}
+}
+impl Drop for Field {
+	fn drop(&mut self) {
+		FIELD_REGISTRY_DEBUG_GIZMOS.remove(self);
+	}
+}
+impl AspectIdentifier for Field {
+	impl_aspect_for_field_aspect_id! {}
+}
+impl Aspect for Field {
+	impl_aspect_for_field_aspect! {}
+}
+impl FieldAspect for Field {
+	fn set_shape(node: Arc<Node>, _calling_client: Arc<Client>, shape: Shape) -> Result<()> {
+		let field = node.get_aspect::<Field>()?;
+		*field.shape.lock() = shape;
+		Ok(())
+	}
+
+	async fn export_field(node: Arc<Node>, _calling_client: Arc<Client>) -> Result<u64> {
+		let id = rand::random();
+		EXPORTED_FIELDS.lock().insert(id, Arc::downgrade(&node));
+		Ok(id)
+	}
+}
+impl FieldTrait for Field {
+	fn spatial_ref(&self) -> &Spatial {
+		&self.spatial
+	}
+	fn local_distance(&self, p: Vec3A) -> f32 {
+		match self.shape.lock().clone() {
+			Shape::Box(size) => {
+				let q = vec3(
+					p.x.abs() - (size.x * 0.5_f32),
+					p.y.abs() - (size.y * 0.5_f32),
+					p.z.abs() - (size.z * 0.5_f32),
+				);
+				let v = vec3a(q.x.max(0_f32), q.y.max(0_f32), q.z.max(0_f32));
+				v.length() + q.x.max(q.y.max(q.z)).min(0_f32)
+			}
+			Shape::Cylinder(CylinderShape { length, radius }) => {
+				let d = vec2(p.xz().length().abs() - radius, p.y.abs() - (length * 0.5));
+				d.x.max(d.y).min(0.0) + d.max(vec2(0.0, 0.0)).length()
+			}
+			Shape::Sphere(radius) => p.length() - radius,
+			Shape::Torus(TorusShape { radius_a, radius_b }) => {
+				let q = vec2(p.xz().length() - radius_a, p.y);
+				q.length() - radius_b
+			}
+		}
+	}
+}
+
+pub struct FieldRef;
+impl AspectIdentifier for FieldRef {
+	impl_aspect_for_field_ref_aspect_id! {}
+}
+impl Aspect for FieldRef {
+	impl_aspect_for_field_ref_aspect! {}
+}
+impl FieldRefAspect for FieldRef {
+	async fn distance(
+		node: Arc<Node>,
+		_calling_client: Arc<Client>,
+		space: Arc<Node>,
+		point: Vector3<f32>,
+	) -> Result<f32> {
+		let reference_space = space.get_aspect::<Spatial>()?;
+		let field = node.get_aspect::<Field>()?;
+		Ok(field.distance(&reference_space, point.into()))
+	}
+
+	async fn normal(
+		node: Arc<Node>,
+		_calling_client: Arc<Client>,
+		space: Arc<Node>,
+		point: Vector3<f32>,
+	) -> Result<Vector3<f32>> {
+		let reference_space = space.get_aspect::<Spatial>()?;
+		let field = node.get_aspect::<Field>()?;
+		Ok(field.normal(&reference_space, point.into(), 0.0001).into())
+	}
+
+	async fn closest_point(
+		node: Arc<Node>,
+		_calling_client: Arc<Client>,
+		space: Arc<Node>,
+		point: Vector3<f32>,
+	) -> Result<Vector3<f32>> {
+		let reference_space = space.get_aspect::<Spatial>()?;
+		let field = node.get_aspect::<Field>()?;
+		Ok(field
+			.closest_point(&reference_space, point.into(), 0.0001)
+			.into())
+	}
+
+	async fn ray_march(
+		node: Arc<Node>,
+		_calling_client: Arc<Client>,
+		space: Arc<Node>,
+		ray_origin: Vector3<f32>,
+		ray_direction: Vector3<f32>,
+	) -> Result<RayMarchResult> {
+		let space = space.get_aspect::<Spatial>()?;
+		let field = node.get_aspect::<Field>()?;
+		Ok(field.ray_march(Ray {
+			origin: ray_origin.into(),
+			direction: ray_direction.into(),
+			space,
+		}))
+	}
+}
+
+impl InterfaceAspect for Interface {
+	async fn import_field_ref(
+		_node: Arc<Node>,
+		calling_client: Arc<Client>,
+		uid: u64,
+	) -> Result<Arc<Node>> {
+		Ok(EXPORTED_FIELDS
+			.lock()
+			.get(&uid)
+			.and_then(|s| s.upgrade())
+			.map(|s| {
+				Alias::create(
+					&s,
+					&calling_client,
+					FIELD_REF_ASPECT_ALIAS_INFO.clone(),
+					None,
+				)
+				.unwrap()
+			})
+			.ok_or_eyre("Couldn't import field with that ID")?)
+	}
+
+	fn create_field(
+		_node: Arc<Node>,
+		calling_client: Arc<Client>,
+		id: u64,
+		parent: Arc<Node>,
+		transform: Transform,
+		shape: Shape,
+	) -> Result<()> {
+		let transform = transform.to_mat4(true, true, false);
+		let parent = parent.get_aspect::<Spatial>()?;
+		let node = Node::from_id(&calling_client, id, true).add_to_scenegraph()?;
+		Spatial::add_to(&node, Some(parent.clone()), transform, false);
+		Field::add_to(&node, shape)?;
+		Ok(())
+	}
+}

src/nodes/fields/box.rs

@@ -1,82 +0,0 @@
-use super::{Field, FieldTrait, Node};
-use crate::core::client::Client;
-use crate::nodes::spatial::{find_spatial_parent, parse_transform, Spatial};
-use anyhow::{ensure, Result};
-use glam::{vec3, vec3a, Vec3, Vec3A};
-use mint::Vector3;
-use parking_lot::Mutex;
-use serde::Deserialize;
-use stardust_xr::schemas::flex::deserialize;
-use stardust_xr::values::Transform;
-use std::sync::Arc;
-
-pub struct BoxField {
-	space: Arc<Spatial>,
-	size: Mutex<Vec3>,
-}
-
-impl BoxField {
-	pub fn add_to(node: &Arc<Node>, size: Vector3<f32>) -> Result<()> {
-		ensure!(
-			node.spatial.get().is_some(),
-			"Internal: Node does not have a spatial attached!"
-		);
-		ensure!(
-			node.field.get().is_none(),
-			"Internal: Node already has a field attached!"
-		);
-		let box_field = BoxField {
-			space: node.spatial.get().unwrap().clone(),
-			size: Mutex::new(size.into()),
-		};
-		box_field.add_field_methods(node);
-		node.add_local_signal("setSize", BoxField::set_size_flex);
-		let _ = node.field.set(Arc::new(Field::Box(box_field)));
-		Ok(())
-	}
-
-	pub fn set_size(&self, size: Vector3<f32>) {
-		*self.size.lock() = size.into();
-	}
-
-	pub fn set_size_flex(node: &Node, _calling_client: Arc<Client>, data: &[u8]) -> Result<()> {
-		if let Field::Box(box_field) = node.field.get().unwrap().as_ref() {
-			box_field.set_size(deserialize(data)?);
-		}
-		Ok(())
-	}
-}
-
-impl FieldTrait for BoxField {
-	fn local_distance(&self, p: Vec3A) -> f32 {
-		let size = self.size.lock();
-		let q = vec3(
-			p.x.abs() - (size.x * 0.5_f32),
-			p.y.abs() - (size.y * 0.5_f32),
-			p.z.abs() - (size.z * 0.5_f32),
-		);
-		let v = vec3a(q.x.max(0_f32), q.y.max(0_f32), q.z.max(0_f32));
-		v.length() + q.x.max(q.y.max(q.z)).min(0_f32)
-	}
-	fn spatial_ref(&self) -> &Spatial {
-		self.space.as_ref()
-	}
-}
-
-pub fn create_box_field_flex(_node: &Node, calling_client: Arc<Client>, data: &[u8]) -> Result<()> {
-	#[derive(Deserialize)]
-	struct CreateFieldInfo<'a> {
-		name: &'a str,
-		parent_path: &'a str,
-		transform: Transform,
-		size: Vector3<f32>,
-	}
-	let info: CreateFieldInfo = deserialize(data)?;
-	let node = Node::create(&calling_client, "/field", info.name, true);
-	let parent = find_spatial_parent(&calling_client, info.parent_path)?;
-	let transform = parse_transform(info.transform, true, true, false)?;
-	let node = node.add_to_scenegraph();
-	Spatial::add_to(&node, Some(parent), transform)?;
-	BoxField::add_to(&node, info.size)?;
-	Ok(())
-}

src/nodes/fields/cylinder.rs

@@ -1,89 +0,0 @@
-use super::{Field, FieldTrait, Node};
-use crate::core::client::Client;
-use crate::nodes::spatial::{find_spatial_parent, parse_transform, Spatial};
-use anyhow::{ensure, Result};
-use glam::{swizzles::*, vec2, Vec3A};
-use portable_atomic::AtomicF32;
-use serde::Deserialize;
-use stardust_xr::schemas::flex::deserialize;
-use stardust_xr::values::Transform;
-
-use std::sync::atomic::Ordering;
-use std::sync::Arc;
-
-pub struct CylinderField {
-	space: Arc<Spatial>,
-	length: AtomicF32,
-	radius: AtomicF32,
-}
-
-impl CylinderField {
-	pub fn add_to(node: &Arc<Node>, length: f32, radius: f32) -> Result<()> {
-		ensure!(
-			node.spatial.get().is_some(),
-			"Internal: Node does not have a spatial attached!"
-		);
-		ensure!(
-			node.field.get().is_none(),
-			"Internal: Node already has a field attached!"
-		);
-		let cylinder_field = CylinderField {
-			space: node.spatial.get().unwrap().clone(),
-			length: AtomicF32::new(length.abs()),
-			radius: AtomicF32::new(radius.abs()),
-		};
-		cylinder_field.add_field_methods(node);
-		node.add_local_signal("setSize", CylinderField::set_size_flex);
-		let _ = node.field.set(Arc::new(Field::Cylinder(cylinder_field)));
-		Ok(())
-	}
-
-	pub fn set_size(&self, length: f32, radius: f32) {
-		self.length.store(length.abs(), Ordering::Relaxed);
-		self.radius.store(radius.abs(), Ordering::Relaxed);
-	}
-
-	pub fn set_size_flex(node: &Node, _calling_client: Arc<Client>, data: &[u8]) -> Result<()> {
-		if let Field::Cylinder(cylinder_field) = node.field.get().unwrap().as_ref() {
-			let (length, radius) = deserialize(data)?;
-			cylinder_field.set_size(length, radius);
-		}
-		Ok(())
-	}
-}
-
-impl FieldTrait for CylinderField {
-	fn local_distance(&self, p: Vec3A) -> f32 {
-		let radius = self.radius.load(Ordering::Relaxed);
-		let length = self.length.load(Ordering::Relaxed);
-		let d = vec2(p.xy().length().abs() - radius, p.z.abs() - (length * 0.5));
-
-		d.x.max(d.y).min(0.0) + d.max(vec2(0.0, 0.0)).length()
-	}
-	fn spatial_ref(&self) -> &Spatial {
-		self.space.as_ref()
-	}
-}
-
-pub fn create_cylinder_field_flex(
-	_node: &Node,
-	calling_client: Arc<Client>,
-	data: &[u8],
-) -> Result<()> {
-	#[derive(Deserialize)]
-	struct CreateFieldInfo<'a> {
-		name: &'a str,
-		parent_path: &'a str,
-		transform: Transform,
-		length: f32,
-		radius: f32,
-	}
-	let info: CreateFieldInfo = deserialize(data)?;
-	let node = Node::create(&calling_client, "/field", info.name, true);
-	let parent = find_spatial_parent(&calling_client, info.parent_path)?;
-	let transform = parse_transform(info.transform, true, true, false)?;
-	let node = node.add_to_scenegraph();
-	Spatial::add_to(&node, Some(parent), transform)?;
-	CylinderField::add_to(&node, dbg!(info.length), dbg!(info.radius))?;
-	Ok(())
-}

src/nodes/fields/mod.rs

@@ -1,212 +0,0 @@
-mod r#box;
-mod cylinder;
-mod sphere;
-
-use self::cylinder::{create_cylinder_field_flex, CylinderField};
-use self::r#box::{create_box_field_flex, BoxField};
-use self::sphere::{create_sphere_field_flex, SphereField};
-
-use super::spatial::Spatial;
-use super::Node;
-use crate::core::client::Client;
-use crate::nodes::spatial::find_reference_space;
-use anyhow::Result;
-use glam::{vec2, vec3a, Vec3, Vec3A};
-use mint::Vector3;
-use serde::Deserialize;
-use stardust_xr::schemas::flex::{deserialize, serialize};
-
-use std::ops::Deref;
-use std::sync::Arc;
-
-pub trait FieldTrait {
-	fn local_distance(&self, p: Vec3A) -> f32;
-	fn local_normal(&self, p: Vec3A, r: f32) -> Vec3A {
-		let d = self.local_distance(p);
-		let e = vec2(r, 0_f32);
-
-		let n = vec3a(d, d, d)
-			- vec3a(
-				self.local_distance(vec3a(e.x, e.y, e.y)),
-				self.local_distance(vec3a(e.y, e.x, e.y)),
-				self.local_distance(vec3a(e.y, e.y, e.x)),
-			);
-
-		n.normalize()
-	}
-	fn local_closest_point(&self, p: Vec3A, r: f32) -> Vec3A {
-		p - (self.local_normal(p, r) * self.local_distance(p))
-	}
-
-	fn distance(&self, reference_space: &Spatial, p: Vec3A) -> f32 {
-		let reference_to_local_space =
-			Spatial::space_to_space_matrix(Some(reference_space), Some(self.spatial_ref()));
-		let local_p = reference_to_local_space.transform_point3a(p);
-		self.local_distance(local_p)
-	}
-	fn normal(&self, reference_space: &Spatial, p: Vec3A, r: f32) -> Vec3A {
-		let reference_to_local_space =
-			Spatial::space_to_space_matrix(Some(reference_space), Some(self.spatial_ref()));
-		let local_p = reference_to_local_space.transform_point3a(p);
-		reference_to_local_space
-			.inverse()
-			.transform_vector3a(self.local_normal(local_p, r))
-	}
-	fn closest_point(&self, reference_space: &Spatial, p: Vec3A, r: f32) -> Vec3A {
-		let reference_to_local_space =
-			Spatial::space_to_space_matrix(Some(reference_space), Some(self.spatial_ref()));
-		let local_p = reference_to_local_space.transform_point3a(p);
-		reference_to_local_space
-			.inverse()
-			.transform_point3a(self.local_closest_point(local_p, r))
-	}
-
-	fn add_field_methods(&self, node: &Arc<Node>) {
-		node.add_local_method("distance", field_distance_flex);
-		node.add_local_method("normal", field_normal_flex);
-		node.add_local_method("closest_point", field_closest_point_flex);
-	}
-
-	fn spatial_ref(&self) -> &Spatial;
-}
-
-fn field_distance_flex(node: &Node, calling_client: Arc<Client>, data: &[u8]) -> Result<Vec<u8>> {
-	#[derive(Deserialize)]
-	struct FieldInfoArgs<'a> {
-		reference_space_path: &'a str,
-		point: Vector3<f32>,
-	}
-	let args: FieldInfoArgs = deserialize(data)?;
-	let reference_space = find_reference_space(&calling_client, args.reference_space_path)?;
-
-	let distance = node
-		.field
-		.get()
-		.unwrap()
-		.distance(reference_space.as_ref(), args.point.into());
-	Ok(serialize(distance)?)
-}
-fn field_normal_flex(node: &Node, calling_client: Arc<Client>, data: &[u8]) -> Result<Vec<u8>> {
-	#[derive(Deserialize)]
-	struct FieldInfoArgs<'a> {
-		reference_space_path: &'a str,
-		point: Vector3<f32>,
-		radius: Option<f32>,
-	}
-	let args: FieldInfoArgs = deserialize(data)?;
-	let reference_space = find_reference_space(&calling_client, args.reference_space_path)?;
-
-	let normal = node.field.get().as_ref().unwrap().normal(
-		reference_space.as_ref(),
-		args.point.into(),
-		args.radius.unwrap_or(0.001),
-	);
-	Ok(serialize(mint::Vector3::from(normal))?)
-}
-fn field_closest_point_flex(
-	node: &Node,
-	calling_client: Arc<Client>,
-	data: &[u8],
-) -> Result<Vec<u8>> {
-	#[derive(Deserialize)]
-	struct FieldInfoArgs<'a> {
-		reference_space_path: &'a str,
-		point: Vector3<f32>,
-		radius: Option<f32>,
-	}
-	let args: FieldInfoArgs = deserialize(data)?;
-	let reference_space = find_reference_space(&calling_client, args.reference_space_path)?;
-
-	let closest_point = node.field.get().as_ref().unwrap().closest_point(
-		reference_space.as_ref(),
-		args.point.into(),
-		args.radius.unwrap_or(0.001),
-	);
-	Ok(serialize(mint::Vector3::from(closest_point))?)
-}
-
-pub enum Field {
-	Box(BoxField),
-	Cylinder(CylinderField),
-	Sphere(SphereField),
-}
-
-impl Deref for Field {
-	type Target = dyn FieldTrait;
-	fn deref(&self) -> &Self::Target {
-		match self {
-			Field::Box(field) => field,
-			Field::Cylinder(field) => field,
-			Field::Sphere(field) => field,
-		}
-	}
-}
-
-pub fn create_interface(client: &Arc<Client>) {
-	let node = Node::create(client, "", "field", false);
-	node.add_local_signal("createBoxField", create_box_field_flex);
-	node.add_local_signal("createCylinderField", create_cylinder_field_flex);
-	node.add_local_signal("createSphereField", create_sphere_field_flex);
-	node.add_to_scenegraph();
-}
-
-pub struct Ray {
-	pub origin: Vec3,
-	pub direction: Vec3,
-	pub space: Arc<Spatial>,
-}
-
-pub struct RayMarchResult {
-	pub ray: Ray,
-	pub distance: f32,
-	pub deepest_point_distance: f32,
-	pub ray_length: f32,
-	pub ray_steps: u32,
-}
-
-// const MIN_RAY_STEPS: u32 = 0;
-const MAX_RAY_STEPS: u32 = 1000;
-
-const MIN_RAY_MARCH: f32 = 0.001_f32;
-const MAX_RAY_MARCH: f32 = f32::MAX;
-
-// const MIN_RAY_LENGTH: f32 = 0_f32;
-const MAX_RAY_LENGTH: f32 = 1000_f32;
-
-pub fn ray_march(ray: Ray, field: &Field) -> RayMarchResult {
-	let mut result = RayMarchResult {
-		ray,
-		distance: f32::MAX,
-		deepest_point_distance: 0_f32,
-		ray_length: 0_f32,
-		ray_steps: 0,
-	};
-
-	let ray_to_field_matrix =
-		Spatial::space_to_space_matrix(Some(&result.ray.space), Some(field.spatial_ref()));
-	let mut ray_point = ray_to_field_matrix.transform_point3a(result.ray.origin.into());
-	let ray_direction = ray_to_field_matrix.transform_vector3a(result.ray.direction.into());
-
-	while result.ray_steps < MAX_RAY_STEPS && result.ray_length < MAX_RAY_LENGTH {
-		let distance = field.local_distance(ray_point);
-		let march_distance = distance.clamp(MIN_RAY_MARCH, MAX_RAY_MARCH);
-
-		result.ray_length += march_distance;
-		ray_point += ray_direction * march_distance;
-
-		if result.distance > distance {
-			result.deepest_point_distance = result.ray_length;
-		}
-		result.distance = distance.min(result.distance);
-
-		result.ray_steps += 1;
-	}
-
-	result
-}
-
-pub fn find_field(client: &Client, path: &str) -> Result<Arc<Field>> {
-	Ok(client
-		.get_node("Field", path)?
-		.get_aspect("Field", "info", |n| &n.field)?)
-}

src/nodes/fields/sphere.rs

@@ -1,90 +0,0 @@
-use super::{Field, FieldTrait, Node};
-use crate::core::client::Client;
-use crate::nodes::spatial::{find_spatial_parent, Spatial};
-use anyhow::{ensure, Result};
-use glam::{Mat4, Vec3A};
-use mint::Vector3;
-use portable_atomic::AtomicF32;
-use serde::Deserialize;
-use stardust_xr::schemas::flex::deserialize;
-use std::sync::atomic::Ordering;
-use std::sync::Arc;
-
-pub struct SphereField {
-	space: Arc<Spatial>,
-	radius: AtomicF32,
-}
-
-impl SphereField {
-	pub fn add_to(node: &Arc<Node>, radius: f32) -> Result<()> {
-		ensure!(
-			node.spatial.get().is_some(),
-			"Internal: Node does not have a spatial attached!"
-		);
-		ensure!(
-			node.field.get().is_none(),
-			"Internal: Node already has a field attached!"
-		);
-		let sphere_field = SphereField {
-			space: node.spatial.get().unwrap().clone(),
-			radius: AtomicF32::new(radius),
-		};
-		sphere_field.add_field_methods(node);
-		node.add_local_signal("setRadius", SphereField::set_radius_flex);
-		let _ = node.field.set(Arc::new(Field::Sphere(sphere_field)));
-		Ok(())
-	}
-
-	pub fn set_radius(&self, radius: f32) {
-		self.radius.store(radius, Ordering::Relaxed);
-	}
-
-	pub fn set_radius_flex(node: &Node, _calling_client: Arc<Client>, data: &[u8]) -> Result<()> {
-		let radius = flexbuffers::Reader::get_root(data)?.get_f64()? as f32;
-		if let Field::Sphere(sphere_field) = node.field.get().unwrap().as_ref() {
-			sphere_field.set_radius(radius);
-		}
-		Ok(())
-	}
-}
-
-impl FieldTrait for SphereField {
-	fn local_distance(&self, p: Vec3A) -> f32 {
-		p.length() - self.radius.load(Ordering::Relaxed)
-	}
-	fn local_normal(&self, p: Vec3A, _r: f32) -> Vec3A {
-		-p.normalize()
-	}
-	fn local_closest_point(&self, p: Vec3A, _r: f32) -> Vec3A {
-		p.normalize() * self.radius.load(Ordering::Relaxed)
-	}
-	fn spatial_ref(&self) -> &Spatial {
-		self.space.as_ref()
-	}
-}
-
-pub fn create_sphere_field_flex(
-	_node: &Node,
-	calling_client: Arc<Client>,
-	data: &[u8],
-) -> Result<()> {
-	#[derive(Deserialize)]
-	struct CreateFieldInfo<'a> {
-		name: &'a str,
-		parent_path: &'a str,
-		origin: Option<Vector3<f32>>,
-		radius: f32,
-	}
-	let info: CreateFieldInfo = deserialize(data)?;
-	let node = Node::create(&calling_client, "/field", info.name, true);
-	let parent = find_spatial_parent(&calling_client, info.parent_path)?;
-	let transform = Mat4::from_translation(
-		info.origin
-			.unwrap_or_else(|| Vector3::from([0.0; 3]))
-			.into(),
-	);
-	let node = node.add_to_scenegraph();
-	Spatial::add_to(&node, Some(parent), transform)?;
-	SphereField::add_to(&node, info.radius)?;
-	Ok(())
-}

src/nodes/hmd.rs

@@ -1,42 +0,0 @@
-use super::{alias::Alias, spatial::Spatial, Node};
-use crate::{
-	core::client::{Client, INTERNAL_CLIENT},
-	nodes::alias::AliasInfo,
-};
-use glam::{vec3, Mat4};
-use std::sync::Arc;
-use stereokit::StereoKit;
-
-lazy_static::lazy_static! {
-	static ref HMD: Arc<Node> = create();
-}
-
-fn create() -> Arc<Node> {
-	let node = Arc::new(Node::create(&INTERNAL_CLIENT, "", "hmd", false));
-	Spatial::add_to(&node, None, Mat4::IDENTITY).unwrap();
-
-	node
-}
-
-pub fn frame(sk: &StereoKit) {
-	let spatial = HMD.spatial.get().unwrap();
-	let hmd_pose = sk.input_head();
-	*spatial.transform.lock() = Mat4::from_scale_rotation_translation(
-		vec3(1.0, 1.0, 1.0),
-		hmd_pose.orientation.into(),
-		hmd_pose.position.into(),
-	);
-}
-
-pub fn make_alias(client: &Arc<Client>) -> Arc<Node> {
-	Alias::new(
-		client,
-		"",
-		"hmd",
-		&HMD,
-		AliasInfo {
-			local_signals: vec!["getTransform"],
-			..Default::default()
-		},
-	)
-}

src/nodes/input/hand.rs

@@ -1,48 +1,89 @@
-use crate::nodes::fields::Field;
+use super::{Finger, Hand, InputDataTrait, InputHandler, InputMethod, Joint, Thumb};
+use crate::nodes::fields::{Field, FieldTrait};
 use crate::nodes::spatial::Spatial;
-use glam::{vec3a, Mat4};
-use stardust_xr::schemas::flat::{Hand as FlatHand, InputDataType, Joint};
+use glam::{Mat4, Quat, vec3a};
 use std::sync::Arc;
 
-use super::{DistanceLink, InputSpecialization};
-
-#[derive(Debug, Default)]
-pub struct Hand {
-	pub base: FlatHand,
+impl Default for Joint {
+	fn default() -> Self {
+		Joint {
+			position: [0.0; 3].into(),
+			rotation: Quat::IDENTITY.into(),
+			radius: 0.0,
+			distance: 0.0,
+		}
+	}
 }
-impl InputSpecialization for Hand {
+#[allow(clippy::derivable_impls)]
+impl Default for Finger {
+	fn default() -> Self {
+		Finger {
+			tip: Default::default(),
+			distal: Default::default(),
+			intermediate: Default::default(),
+			proximal: Default::default(),
+			metacarpal: Default::default(),
+		}
+	}
+}
+#[allow(clippy::derivable_impls)]
+impl Default for Thumb {
+	fn default() -> Self {
+		Thumb {
+			tip: Default::default(),
+			distal: Default::default(),
+			proximal: Default::default(),
+			metacarpal: Default::default(),
+		}
+	}
+}
+#[allow(clippy::derivable_impls)]
+impl Default for Hand {
+	fn default() -> Self {
+		Hand {
+			right: Default::default(),
+			thumb: Default::default(),
+			index: Default::default(),
+			middle: Default::default(),
+			ring: Default::default(),
+			little: Default::default(),
+			palm: Default::default(),
+			wrist: Default::default(),
+			elbow: Default::default(),
+		}
+	}
+}
+
+impl InputDataTrait for Hand {
 	fn distance(&self, space: &Arc<Spatial>, field: &Field) -> f32 {
 		let mut min_distance = f32::MAX;
 
 		for tip in [
-			&self.base.thumb.tip.position,
-			&self.base.index.tip.position,
-			&self.base.middle.tip.position,
-			&self.base.ring.tip.position,
-			&self.base.little.tip.position,
+			&self.thumb.tip.position,
+			&self.index.tip.position,
+			&self.middle.tip.position,
+			&self.ring.tip.position,
+			&self.little.tip.position,
 		] {
 			min_distance = min_distance.min(field.distance(space, vec3a(tip.x, tip.y, tip.z)));
 		}
 
 		min_distance
 	}
-	fn serialize(
-		&self,
-		_distance_link: &DistanceLink,
-		local_to_handler_matrix: Mat4,
-	) -> InputDataType {
-		let mut hand = self.base;
+	fn transform(&mut self, method: &InputMethod, handler: &InputHandler) {
+		let local_to_handler_matrix =
+			Spatial::space_to_space_matrix(Some(&method.spatial), Some(&handler.spatial));
 		let mut joints: Vec<&mut Joint> = Vec::new();
 
-		joints.extend([&mut hand.palm, &mut hand.wrist]);
-		if let Some(elbow) = &mut hand.elbow {
+		joints.extend([&mut self.palm, &mut self.wrist]);
+		if let Some(elbow) = &mut self.elbow {
 			joints.push(elbow);
 		}
 		for finger in [
-			&mut hand.index,
-			&mut hand.middle,
-			&mut hand.ring,
-			&mut hand.little,
+			&mut self.index,
+			&mut self.middle,
+			&mut self.ring,
+			&mut self.little,
 		] {
 			joints.extend([
 				&mut finger.tip,
@@ -53,21 +94,19 @@ impl InputSpecialization for Hand {
 			]);
 		}
 		joints.extend([
-			&mut hand.thumb.tip,
-			&mut hand.thumb.distal,
-			&mut hand.thumb.proximal,
-			&mut hand.thumb.metacarpal,
+			&mut self.thumb.tip,
+			&mut self.thumb.distal,
+			&mut self.thumb.proximal,
+			&mut self.thumb.metacarpal,
 		]);
 
 		for joint in joints {
-			let joint_matrix =
-				Mat4::from_rotation_translation(joint.rotation.into(), joint.position.into())
-					* local_to_handler_matrix;
+			let joint_matrix = local_to_handler_matrix
+				* Mat4::from_rotation_translation(joint.rotation.into(), joint.position.into());
 			let (_, rotation, position) = joint_matrix.to_scale_rotation_translation();
 			joint.position = position.into();
 			joint.rotation = rotation.into();
+			joint.distance = handler.field.distance(&handler.spatial, position.into());
 		}
-
-		InputDataType::Hand(Box::new(hand))
 	}
 }

src/nodes/input/handler.rs

@@ -0,0 +1,39 @@
+use super::{INPUT_HANDLER_REGISTRY, INPUT_METHOD_REGISTRY, InputHandlerAspect};
+use crate::nodes::{Node, alias::AliasList, fields::Field, spatial::Spatial};
+use color_eyre::eyre::Result;
+use std::sync::Arc;
+
+pub struct InputHandler {
+	pub spatial: Arc<Spatial>,
+	pub field: Arc<Field>,
+	pub(super) method_aliases: AliasList,
+}
+impl InputHandler {
+	pub fn add_to(node: &Arc<Node>, field: &Arc<Field>) -> Result<()> {
+		let handler = InputHandler {
+			spatial: node.get_aspect::<Spatial>().unwrap().clone(),
+			field: field.clone(),
+			method_aliases: AliasList::default(),
+		};
+		for method in INPUT_METHOD_REGISTRY.get_valid_contents() {
+			method.handle_new_handler(&handler);
+		}
+		let handler = INPUT_HANDLER_REGISTRY.add(handler);
+		node.add_aspect_raw(handler);
+		Ok(())
+	}
+}
+impl InputHandlerAspect for InputHandler {}
+impl PartialEq for InputHandler {
+	fn eq(&self, other: &Self) -> bool {
+		self.spatial == other.spatial
+	}
+}
+impl Drop for InputHandler {
+	fn drop(&mut self) {
+		INPUT_HANDLER_REGISTRY.remove(self);
+		for method in INPUT_METHOD_REGISTRY.get_valid_contents() {
+			method.handle_drop_handler(self);
+		}
+	}
+}

src/nodes/input/method.rs

@@ -0,0 +1,279 @@
+use super::{
+	INPUT_HANDLER_REGISTRY, INPUT_METHOD_REF_ASPECT_ALIAS_INFO, INPUT_METHOD_REGISTRY, InputData,
+	InputDataTrait, InputDataType, InputHandler, InputMethodAspect, InputMethodRefAspect,
+	input_method_client,
+};
+use crate::{
+	core::{
+		client::Client,
+		error::{Result, ServerError},
+		registry::Registry,
+	},
+	nodes::{
+		Node,
+		alias::{Alias, AliasList},
+		fields::{FIELD_ALIAS_INFO, Field},
+		spatial::Spatial,
+	},
+};
+use color_eyre::eyre::eyre;
+use parking_lot::Mutex;
+use stardust_xr::values::Datamap;
+use std::sync::{Arc, Weak};
+
+pub struct InputMethod {
+	pub spatial: Arc<Spatial>,
+	pub data: Mutex<InputDataType>,
+	pub datamap: Mutex<Datamap>,
+
+	handler_aliases: AliasList,
+	handler_field_aliases: AliasList,
+	pub(super) handler_order: Mutex<Vec<Weak<InputHandler>>>,
+	pub capture_attempts: Registry<InputHandler>,
+	pub captures: Registry<InputHandler>,
+}
+impl InputMethod {
+	pub fn add_to(
+		node: &Arc<Node>,
+		data: InputDataType,
+		datamap: Datamap,
+	) -> Result<Arc<InputMethod>> {
+		let method = InputMethod {
+			spatial: node.get_aspect::<Spatial>().unwrap().clone(),
+			data: Mutex::new(data),
+			datamap: Mutex::new(datamap),
+
+			handler_aliases: AliasList::default(),
+			handler_field_aliases: AliasList::default(),
+			handler_order: Mutex::new(Vec::new()),
+			capture_attempts: Registry::new(),
+			captures: Registry::new(),
+		};
+		for handler in INPUT_HANDLER_REGISTRY.get_valid_contents() {
+			method.handle_new_handler(&handler);
+		}
+		let method = INPUT_METHOD_REGISTRY.add(method);
+		node.add_aspect_raw(method.clone());
+		node.add_aspect(InputMethodRef);
+		Ok(method)
+	}
+
+	pub fn distance(&self, to: &Field) -> f32 {
+		self.data.lock().distance(&self.spatial, to)
+	}
+
+	pub fn set_handler_order<'a>(&self, handlers: impl Iterator<Item = &'a Arc<InputHandler>>) {
+		*self.handler_order.lock() = handlers.map(Arc::downgrade).collect();
+	}
+
+	pub(super) fn make_alias(&self, handler: &InputHandler) {
+		let Some(method_node) = self.spatial.node() else {
+			return;
+		};
+		let Some(handler_node) = handler.spatial.node() else {
+			return;
+		};
+		let Some(client) = handler_node.get_client() else {
+			return;
+		};
+		let Ok(method_alias) = Alias::create(
+			&method_node,
+			&client,
+			INPUT_METHOD_REF_ASPECT_ALIAS_INFO.clone(),
+			Some(&handler.method_aliases),
+		) else {
+			return;
+		};
+		method_alias.set_enabled(false);
+	}
+	pub(super) fn handle_new_handler(&self, handler: &InputHandler) {
+		self.make_alias(handler);
+
+		let Some(method_node) = self.spatial.node() else {
+			return;
+		};
+		let Some(method_client) = method_node.get_client() else {
+			return;
+		};
+		let Some(handler_node) = handler.spatial.node() else {
+			return;
+		};
+		// Receiver itself
+		let Ok(handler_alias) = Alias::create(
+			&handler_node,
+			&method_client,
+			INPUT_METHOD_REF_ASPECT_ALIAS_INFO.clone(),
+			Some(&self.handler_aliases),
+		) else {
+			return;
+		};
+
+		let Some(handler_field_node) = handler.field.spatial.node() else {
+			return;
+		};
+		// Handler's field
+		let Ok(rx_field_alias) = Alias::create(
+			&handler_field_node,
+			&method_client,
+			FIELD_ALIAS_INFO.clone(),
+			Some(&self.handler_field_aliases),
+		) else {
+			return;
+		};
+
+		let _ = input_method_client::create_handler(&method_node, &handler_alias, &rx_field_alias);
+	}
+	pub(super) fn handle_drop_handler(&self, handler: &InputHandler) {
+		let Some(tx_node) = self.spatial.node() else {
+			return;
+		};
+		let Some(handler_alias) = self.handler_aliases.get_from_aspect(handler) else {
+			return;
+		};
+		let _ = input_method_client::destroy_handler(&tx_node, handler_alias.id);
+		self.handler_aliases.remove_aspect(handler);
+		self.handler_field_aliases
+			.remove_aspect(handler.field.as_ref());
+		self.capture_attempts.remove(handler);
+	}
+
+	pub(super) fn serialize(&self, alias_id: u64, handler: &Arc<InputHandler>) -> InputData {
+		let mut input = self.data.lock().clone();
+		input.transform(self, handler);
+
+		InputData {
+			id: alias_id,
+			input,
+			distance: self.distance(&handler.field),
+			datamap: self.datamap.lock().clone(),
+			order: self
+				.handler_order
+				.lock()
+				.iter()
+				.enumerate()
+				.find(|(_, h)| h.ptr_eq(&Arc::downgrade(handler)))
+				.unwrap()
+				.0 as u32,
+			captured: self.captures.get_valid_contents().contains(handler),
+		}
+	}
+
+	pub(super) fn cull_capture_attempts(&self) {
+		let sent = self
+			.handler_order
+			.lock()
+			.iter()
+			.filter_map(Weak::upgrade)
+			.collect::<Registry<InputHandler>>();
+		self.capture_attempts.retain(|handler| {
+			!handler
+				.spatial
+				.node()
+				.and_then(|n| n.get_client())
+				.map(|c| c.unresponsive())
+				.unwrap_or(false)
+				&& sent.contains(handler)
+		});
+	}
+}
+impl InputMethodAspect for InputMethod {
+	#[doc = "Set the spatial input component of this input method. You must keep the same input data type throughout the entire thing."]
+	fn set_input(
+		node: Arc<Node>,
+		_calling_client: Arc<Client>,
+		input: InputDataType,
+	) -> Result<()> {
+		let input_method = node.get_aspect::<InputMethod>()?;
+		*input_method.data.lock() = input;
+		Ok(())
+	}
+
+	#[doc = "Set the datmap of this input method"]
+	fn set_datamap(node: Arc<Node>, _calling_client: Arc<Client>, datamap: Datamap) -> Result<()> {
+		let input_method = node.get_aspect::<InputMethod>()?;
+		*input_method.datamap.lock() = datamap;
+		Ok(())
+	}
+
+	#[doc = "Manually set the order of handlers to propagate input to, or else let the server decide."]
+	fn set_handler_order(
+		node: Arc<Node>,
+		_calling_client: Arc<Client>,
+		handlers: Vec<Arc<Node>>,
+	) -> Result<()> {
+		let input_method = node.get_aspect::<InputMethod>()?;
+		let handlers = handlers
+			.into_iter()
+			.filter_map(|p| p.get_aspect::<InputHandler>().ok())
+			.map(|i| Arc::downgrade(&i))
+			.collect::<Vec<_>>();
+
+		*input_method.handler_order.lock() = handlers;
+		Ok(())
+	}
+
+	#[doc = "Set which handlers are captured."]
+	fn set_captures(
+		node: Arc<Node>,
+		_calling_client: Arc<Client>,
+		handlers: Vec<Arc<Node>>,
+	) -> Result<()> {
+		let input_method = node.get_aspect::<InputMethod>()?;
+		input_method.captures.clear();
+		for handler in handlers {
+			let Ok(handler) = handler.get_aspect::<InputHandler>() else {
+				continue;
+			};
+			input_method.captures.add_raw(&handler);
+		}
+		Ok(())
+	}
+}
+impl Drop for InputMethod {
+	fn drop(&mut self) {
+		INPUT_METHOD_REGISTRY.remove(self);
+	}
+}
+
+pub struct InputMethodRef;
+impl InputMethodRefAspect for InputMethodRef {
+	#[doc = "Try to capture the input method with the given handler. When the handler does not get input from the method, it will be released."]
+	fn try_capture(
+		node: Arc<Node>,
+		_calling_client: Arc<Client>,
+		handler: Arc<Node>,
+	) -> Result<()> {
+		let input_method = node.get_aspect::<InputMethod>()?;
+		let input_handler = handler.get_aspect::<InputHandler>()?;
+
+		input_method.capture_attempts.add_raw(&input_handler);
+
+		let Some(handler_alias) = input_method
+			.handler_aliases
+			.get_from_aspect(&*input_handler)
+		else {
+			return Err(ServerError::Report(eyre!(
+				"Internal: Couldn't get handler alias somehow?"
+			)));
+		};
+		input_method_client::request_capture_handler(&node, handler_alias.get_id())
+	}
+
+	#[doc = "If captured by this handler, release it (e.g. the object is let go of after grabbing)."]
+	fn release(node: Arc<Node>, _calling_client: Arc<Client>, handler: Arc<Node>) -> Result<()> {
+		let input_method = node.get_aspect::<InputMethod>()?;
+		let input_handler = handler.get_aspect::<InputHandler>()?;
+
+		input_method.capture_attempts.remove(&input_handler);
+
+		let Some(handler_alias) = input_method
+			.handler_aliases
+			.get_from_aspect(&*input_handler)
+		else {
+			return Err(ServerError::Report(eyre!(
+				"Internal: Couldn't get handler alias somehow?"
+			)));
+		};
+		input_method_client::release_handler(&node, handler_alias.get_id())
+	}
+}

src/nodes/input/mod.rs

@@ -1,278 +1,182 @@
-pub mod hand;
-pub mod pointer;
-pub mod tip;
+#![allow(clippy::needless_question_mark)]
 
-use self::hand::Hand;
-use self::pointer::Pointer;
-use self::tip::Tip;
+mod hand;
+mod handler;
+mod method;
+mod pointer;
+mod tip;
 
+use bevy::tasks::ComputeTaskPool;
+use bevy::tasks::ParallelSlice;
+pub use handler::*;
+pub use method::*;
+use tracing::debug_span;
+
+use super::Aspect;
+use super::AspectIdentifier;
 use super::fields::Field;
-use super::spatial::{find_spatial_parent, parse_transform, Spatial};
-use super::Node;
-use crate::core::client::Client;
-use crate::core::eventloop::FRAME;
-use crate::core::registry::Registry;
-use crate::nodes::fields::find_field;
-use anyhow::{ensure, Result};
-use glam::Mat4;
-use nanoid::nanoid;
-use parking_lot::Mutex;
-use serde::Deserialize;
-use stardust_xr::schemas::flat::{Datamap, InputDataType};
-use stardust_xr::schemas::{flat::InputData, flex::deserialize};
-use stardust_xr::values::Transform;
-use std::ops::Deref;
-use std::sync::atomic::Ordering;
-use std::sync::{Arc, Weak};
+use super::spatial::Spatial;
+use crate::core::error::Result;
+use crate::nodes::spatial::SPATIAL_ASPECT_ALIAS_INFO;
+use crate::nodes::spatial::SPATIAL_REF_ASPECT_ALIAS_INFO;
+use crate::{core::client::Client, nodes::Node};
+use crate::{core::registry::Registry, nodes::spatial::Transform};
+use stardust_xr::values::Datamap;
+use std::sync::Arc;
 
 static INPUT_METHOD_REGISTRY: Registry<InputMethod> = Registry::new();
-static INPUT_HANDLER_REGISTRY: Registry<InputHandler> = Registry::new();
+pub static INPUT_HANDLER_REGISTRY: Registry<InputHandler> = Registry::new();
 
-pub trait InputSpecialization: Send + Sync {
+stardust_xr_server_codegen::codegen_input_protocol!();
+
+impl AspectIdentifier for InputHandler {
+	impl_aspect_for_input_handler_aspect_id! {}
+}
+impl Aspect for InputHandler {
+	impl_aspect_for_input_handler_aspect! {}
+}
+impl AspectIdentifier for InputMethod {
+	impl_aspect_for_input_method_aspect_id! {}
+}
+impl Aspect for InputMethod {
+	impl_aspect_for_input_method_aspect! {}
+}
+impl AspectIdentifier for InputMethodRef {
+	impl_aspect_for_input_method_ref_aspect_id! {}
+}
+impl Aspect for InputMethodRef {
+	impl_aspect_for_input_method_ref_aspect! {}
+}
+
+pub trait InputDataTrait {
+	fn transform(&mut self, method: &InputMethod, handler: &InputHandler);
 	fn distance(&self, space: &Arc<Spatial>, field: &Field) -> f32;
-	fn serialize(
-		&self,
-		distance_link: &DistanceLink,
-		local_to_handler_matrix: Mat4,
-	) -> InputDataType;
 }
-pub enum InputType {
-	Pointer(Pointer),
-	Hand(Box<Hand>),
-	Tip(Tip),
-}
-impl Deref for InputType {
-	type Target = dyn InputSpecialization;
-	fn deref(&self) -> &Self::Target {
+impl InputDataTrait for InputDataType {
+	fn transform(&mut self, method: &InputMethod, handler: &InputHandler) {
 		match self {
-			InputType::Pointer(p) => p,
-			InputType::Hand(h) => h.as_ref(),
-			InputType::Tip(t) => t,
+			InputDataType::Pointer(i) => i.transform(method, handler),
+			InputDataType::Hand(i) => i.transform(method, handler),
+			InputDataType::Tip(i) => i.transform(method, handler),
+		}
+	}
+
+	fn distance(&self, space: &Arc<Spatial>, field: &Field) -> f32 {
+		match self {
+			InputDataType::Pointer(i) => i.distance(space, field),
+			InputDataType::Hand(i) => i.distance(space, field),
+			InputDataType::Tip(i) => i.distance(space, field),
 		}
 	}
 }
 
-pub struct InputMethod {
-	pub uid: String,
-	pub enabled: Mutex<bool>,
-	pub spatial: Arc<Spatial>,
-	pub specialization: Mutex<InputType>,
-	pub captures: Registry<InputHandler>,
-	pub datamap: Mutex<Option<Datamap>>,
-}
-impl InputMethod {
-	pub fn new(spatial: Arc<Spatial>, specialization: InputType) -> Arc<InputMethod> {
-		let method = InputMethod {
-			uid: nanoid!(),
-			enabled: Mutex::new(true),
-			spatial,
-			specialization: Mutex::new(specialization),
-			captures: Registry::new(),
-			datamap: Mutex::new(None),
-		};
-		INPUT_METHOD_REGISTRY.add(method)
-	}
-	#[allow(dead_code)]
-	pub fn add_to(
-		node: &Arc<Node>,
-		specialization: InputType,
-		datamap: Option<Datamap>,
-	) -> Result<()> {
-		ensure!(
-			node.spatial.get().is_some(),
-			"Internal: Node does not have a spatial attached!"
-		);
-
-		node.add_local_signal("setDatamap", InputMethod::set_datamap);
-
-		let method = InputMethod {
-			uid: node.uid.clone(),
-			enabled: Mutex::new(true),
-			spatial: node.spatial.get().unwrap().clone(),
-			specialization: Mutex::new(specialization),
-			captures: Registry::new(),
-			datamap: Mutex::new(datamap),
-		};
-		let method = INPUT_METHOD_REGISTRY.add(method);
-		let _ = node.input_method.set(method);
-		Ok(())
-	}
-
-	fn set_datamap(node: &Node, _calling_client: Arc<Client>, data: &[u8]) -> Result<()> {
-		node.input_method
-			.get()
-			.unwrap()
-			.datamap
-			.lock()
-			.replace(Datamap::new(data.to_vec())?);
-		Ok(())
-	}
-
-	fn distance(&self, to: &Field) -> f32 {
-		self.specialization.lock().distance(&self.spatial, to)
-	}
-}
-impl Drop for InputMethod {
-	fn drop(&mut self) {
-		INPUT_METHOD_REGISTRY.remove(self);
-	}
-}
-
-pub struct DistanceLink {
-	pub distance: f32,
-	pub method: Arc<InputMethod>,
-	pub handler: Arc<InputHandler>,
-	pub handler_field: Arc<Field>,
-}
-impl DistanceLink {
-	fn from(method: Arc<InputMethod>, handler: Arc<InputHandler>) -> Option<Self> {
-		let handler_field = handler.field.upgrade()?;
-		Some(DistanceLink {
-			distance: method.distance(&handler_field),
-			method,
-			handler,
-			handler_field,
-		})
-	}
-
-	fn send_input(&self, frame: u64, datamap: Datamap) {
-		self.handler.send_input(frame, self, datamap);
-	}
-	fn serialize(&self, datamap: Datamap) -> Vec<u8> {
-		let input = self.method.specialization.lock().serialize(
-			self,
-			Spatial::space_to_space_matrix(Some(&self.method.spatial), Some(&self.handler.spatial)),
-		);
-
-		let root = InputData {
-			uid: self.method.uid.clone(),
-			input,
-			distance: self.distance,
-			datamap,
-		};
-		root.serialize()
-	}
-}
-
-pub struct InputHandler {
-	node: Weak<Node>,
-	spatial: Arc<Spatial>,
-	pub field: Weak<Field>,
-}
-impl InputHandler {
-	pub fn add_to(node: &Arc<Node>, field: &Arc<Field>) -> Result<()> {
-		ensure!(
-			node.spatial.get().is_some(),
-			"Internal: Node does not have a spatial attached!"
-		);
-
-		let handler = InputHandler {
-			node: Arc::downgrade(node),
-			spatial: node.spatial.get().unwrap().clone(),
-			field: Arc::downgrade(field),
-		};
-		let handler = INPUT_HANDLER_REGISTRY.add(handler);
-		let _ = node.input_handler.set(handler);
-		Ok(())
-	}
-
-	fn send_input(&self, frame: u64, distance_link: &DistanceLink, datamap: Datamap) {
-		let data = distance_link.serialize(datamap);
-		let node = self.node.upgrade().unwrap();
-		let method = Arc::downgrade(&distance_link.method);
-		let handler = Arc::downgrade(&distance_link.handler);
-
-		tokio::spawn(async move {
-			let data = node.execute_remote_method("input", data).await;
-			if frame == FRAME.load(Ordering::Relaxed) {
-				if let Ok(data) = data {
-					let capture = flexbuffers::Reader::get_root(data.as_slice())
-						.and_then(|data| data.get_bool())
-						.unwrap_or(false);
-
-					if let Some(method) = method.upgrade() {
-						if let Some(handler) = handler.upgrade() {
-							if capture {
-								method.captures.add_raw(&handler);
-							}
-						}
-					}
-				}
-			}
-		});
-	}
-}
-impl Drop for InputHandler {
-	fn drop(&mut self) {
-		INPUT_HANDLER_REGISTRY.remove(self);
-	}
-}
-
-pub fn create_interface(client: &Arc<Client>) {
-	let node = Node::create(client, "", "input", false);
-	node.add_local_signal("createInputHandler", create_input_handler_flex);
-	node.add_local_signal("createInputMethodTip", tip::create_tip_flex);
-	node.add_to_scenegraph();
-}
-
-pub fn create_input_handler_flex(
-	_node: &Node,
-	calling_client: Arc<Client>,
-	data: &[u8],
-) -> Result<()> {
-	#[derive(Deserialize)]
-	struct CreateInputHandlerInfo<'a> {
-		name: &'a str,
-		parent_path: &'a str,
+impl InterfaceAspect for Interface {
+	#[doc = "Create an input method node"]
+	fn create_input_method(
+		_node: Arc<Node>,
+		calling_client: Arc<Client>,
+		id: u64,
+		parent: Arc<Node>,
 		transform: Transform,
-		field_path: &'a str,
-	}
-	let info: CreateInputHandlerInfo = deserialize(data)?;
-	let parent = find_spatial_parent(&calling_client, info.parent_path)?;
-	let transform = parse_transform(info.transform, true, true, false)?;
-	let field = find_field(&calling_client, info.field_path)?;
+		initial_data: InputDataType,
+		datamap: Datamap,
+	) -> Result<()> {
+		let parent = parent.get_aspect::<Spatial>()?;
+		let transform = transform.to_mat4(true, true, true);
 
-	let node = Node::create(&calling_client, "/input/handler", info.name, true).add_to_scenegraph();
-	Spatial::add_to(&node, Some(parent), transform)?;
-	InputHandler::add_to(&node, &field)?;
-	Ok(())
+		let node = Node::from_id(&calling_client, id, true).add_to_scenegraph()?;
+		Spatial::add_to(&node, Some(parent.clone()), transform, false);
+		InputMethod::add_to(&node, initial_data, datamap)?;
+		Ok(())
+	}
+
+	#[doc = "Create an input handler node"]
+	fn create_input_handler(
+		_node: Arc<Node>,
+		calling_client: Arc<Client>,
+		id: u64,
+		parent: Arc<Node>,
+		transform: Transform,
+		field: Arc<Node>,
+	) -> Result<()> {
+		let parent = parent.get_aspect::<Spatial>()?;
+		let transform = transform.to_mat4(true, true, true);
+		let field = field.get_aspect::<Field>()?;
+
+		let node = Node::from_id(&calling_client, id, true).add_to_scenegraph()?;
+		Spatial::add_to(&node, Some(parent.clone()), transform, false);
+		InputHandler::add_to(&node, &field)?;
+		Ok(())
+	}
 }
 
+#[tracing::instrument(level = "debug")]
 pub fn process_input() {
-	for method in INPUT_METHOD_REGISTRY
+	// Iterate over all valid input methods
+	let methods = INPUT_METHOD_REGISTRY
 		.get_valid_contents()
 		.into_iter()
-		.filter(|method| *method.enabled.lock())
-		.filter(|method| method.datamap.lock().is_some())
-	{
-		let mut distance_links: Vec<DistanceLink> = INPUT_HANDLER_REGISTRY
-			.get_valid_contents()
-			.into_iter()
-			.filter(|handler| handler.field.upgrade().is_some())
-			.filter_map(|handler| DistanceLink::from(method.clone(), handler))
-			.collect();
-		distance_links.sort_unstable_by(|a, b| {
-			a.distance
-				.abs()
-				.partial_cmp(&b.distance.abs())
-				.unwrap()
-				.reverse()
+		.filter(|method| {
+			let Some(node) = method.spatial.node() else {
+				return false;
+			};
+			node.enabled()
 		});
+	INPUT_HANDLER_REGISTRY
+		.get_valid_contents()
+		.into_iter()
+		.par_splat_map(ComputeTaskPool::get(), None, |_, handlers| {
+			for handler in handlers {
+				let _span = debug_span!("handle input handler").entered();
+				for method_alias in handler.method_aliases.get_aliases() {
+					method_alias.set_enabled(false);
+				}
 
-		let mut last_distance = 0.0;
-		let frame = FRAME.load(Ordering::Relaxed);
-		let captures = method.captures.get_valid_contents();
-		for distance_link in distance_links {
-			distance_link.send_input(frame, method.datamap.lock().clone().unwrap());
-			if last_distance != distance_link.distance
-				&& captures
-					.iter()
-					.any(|c| Arc::ptr_eq(c, &distance_link.handler))
-			{
-				break;
+				let Some(handler_node) = handler.spatial.node() else {
+					continue;
+				};
+				if !handler_node.enabled() {
+					continue;
+				}
+				if let Some(handler_field_node) = handler.field.spatial.node() {
+					if !handler_field_node.enabled() {
+						continue;
+					}
+				};
+
+				let ser_span = debug_span!("serializing input").entered();
+				let (methods, datas) = methods
+					.clone()
+					// filter out methods without the handler in their handler order
+					.filter(|a| {
+						a.handler_order
+							.lock()
+							.iter()
+							.any(|h| h.ptr_eq(&Arc::downgrade(handler)))
+					})
+					// filter out methods without the proper alias
+					.filter_map(|m| {
+						Some((
+							handler
+								.method_aliases
+								.get_from_original_node(m.spatial.node.clone())?,
+							m,
+						))
+					})
+					// make sure the input method alias is enabled
+					.inspect(|(a, _)| {
+						a.set_enabled(true);
+					})
+					// serialize the data
+					.map(|(a, m)| (a.clone(), m.serialize(a.get_id(), handler)))
+					.unzip::<_, _, Vec<_>, Vec<_>>();
+				drop(ser_span);
+
+				let _span = debug_span!("client input").entered();
+				let _ = input_handler_client::input(&handler_node, &methods, &datas);
 			}
-			last_distance = distance_link.distance;
-		}
-		method.captures.clear();
+		});
+	for method in methods {
+		method.cull_capture_attempts();
 	}
 }

src/nodes/input/pointer.rs

@@ -1,54 +1,52 @@
-use super::{DistanceLink, InputSpecialization};
-use crate::nodes::fields::{ray_march, Field, Ray, RayMarchResult};
-use crate::nodes::spatial::Spatial;
-use glam::{vec3, Mat4};
-use stardust_xr::schemas::flat::{InputDataType, Pointer as FlatPointer};
-use std::sync::Arc;
+use super::{InputDataTrait, InputHandler, InputMethod, Pointer};
+use crate::nodes::{
+	fields::{Field, FieldTrait, Ray, RayMarchResult},
+	spatial::Spatial,
+};
+use glam::{Mat4, Quat, vec3};
+use std::sync::{Arc, Weak};
 
-#[derive(Default)]
-pub struct Pointer {}
-// impl Default for Pointer {
-// 	fn default() -> Self {
-// 		Pointer {
-// 			grab: Default::default(),
-// 			select: Default::default(),
-// 		}
-// 	}
-// }
-impl Pointer {
-	fn ray_march(&self, space: &Arc<Spatial>, field: &Field) -> RayMarchResult {
-		ray_march(
-			Ray {
-				origin: vec3(0_f32, 0_f32, 0_f32),
-				direction: vec3(0_f32, 0_f32, 1_f32),
-				space: space.clone(),
-			},
-			field,
-		)
+impl Default for Pointer {
+	fn default() -> Self {
+		Pointer {
+			origin: [0.0; 3].into(),
+			orientation: Quat::IDENTITY.into(),
+			deepest_point: [0.0; 3].into(),
+		}
 	}
 }
-
-impl InputSpecialization for Pointer {
-	fn distance(&self, space: &Arc<Spatial>, field: &Field) -> f32 {
-		self.ray_march(space, field).distance
-	}
-	fn serialize(
-		&self,
-		distance_link: &DistanceLink,
-		local_to_handler_matrix: Mat4,
-	) -> InputDataType {
-		let (_, orientation, origin) = local_to_handler_matrix.to_scale_rotation_translation();
-		let direction = local_to_handler_matrix.transform_vector3(vec3(0_f32, 0_f32, 1_f32));
-		let ray_march = self.ray_march(
-			&distance_link.method.spatial,
-			&distance_link.handler.field.upgrade().unwrap(),
-		);
-		let deepest_point = (direction * ray_march.deepest_point_distance) + origin;
-
-		InputDataType::Pointer(FlatPointer {
-			origin: origin.into(),
-			orientation: orientation.into(),
-			deepest_point: deepest_point.into(),
+impl Pointer {
+	fn ray_march(&self, method_space: &Arc<Spatial>, field: &Field) -> RayMarchResult {
+		field.ray_march(Ray {
+			origin: vec3(0.0, 0.0, 0.0),
+			direction: vec3(0.0, 0.0, -1.0),
+			space: Spatial::new(
+				Weak::new(),
+				Some(method_space.clone()),
+				Mat4::from_rotation_translation(self.orientation.into(), self.origin.into()),
+			),
 		})
 	}
 }
+impl InputDataTrait for Pointer {
+	fn distance(&self, space: &Arc<Spatial>, field: &Field) -> f32 {
+		let ray_info = self.ray_march(space, field);
+		ray_info.min_distance
+	}
+	fn transform(&mut self, method: &InputMethod, handler: &InputHandler) {
+		let local_to_handler_matrix =
+			Mat4::from_rotation_translation(self.orientation.into(), self.origin.into())
+				* Spatial::space_to_space_matrix(Some(&method.spatial), Some(&handler.spatial));
+		let (_, orientation, origin) = local_to_handler_matrix.to_scale_rotation_translation();
+
+		let ray_march = self.ray_march(&method.spatial, &handler.field);
+		let direction = local_to_handler_matrix
+			.transform_vector3(vec3(0.0, 0.0, -1.0))
+			.normalize();
+		let deepest_point = (direction * ray_march.deepest_point_distance) + origin;
+
+		self.origin = origin.into();
+		self.orientation = orientation.into();
+		self.deepest_point = deepest_point.into();
+	}
+}

src/nodes/input/tip.rs

@@ -1,70 +1,29 @@
-use super::{DistanceLink, InputSpecialization};
-use crate::core::client::Client;
-use crate::nodes::fields::Field;
-use crate::nodes::input::{InputMethod, InputType};
-use crate::nodes::spatial::{find_spatial_parent, parse_transform, Spatial};
-use crate::nodes::Node;
-use anyhow::Result;
-use glam::{vec3a, Mat4};
-use serde::Deserialize;
-use stardust_xr::schemas::flat::{Datamap, InputDataType, Tip as FlatTip};
-use stardust_xr::schemas::flex::deserialize;
-use stardust_xr::values::Transform;
+use super::{InputDataTrait, InputHandler, InputMethod, Tip};
+use crate::nodes::{
+	fields::{Field, FieldTrait},
+	spatial::Spatial,
+};
+use glam::{Mat4, Quat};
 use std::sync::Arc;
 
-#[derive(Default)]
-pub struct Tip {
-	pub radius: f32,
-}
-impl Tip {
-	fn set_radius(node: &Node, _calling_client: Arc<Client>, data: &[u8]) -> Result<()> {
-		if let InputType::Tip(tip) = &mut *node.input_method.get().unwrap().specialization.lock() {
-			tip.radius = deserialize(data)?;
+impl Default for Tip {
+	fn default() -> Self {
+		Tip {
+			origin: [0.0; 3].into(),
+			orientation: Quat::IDENTITY.into(),
 		}
-		Ok(())
 	}
 }
-impl InputSpecialization for Tip {
+impl InputDataTrait for Tip {
 	fn distance(&self, space: &Arc<Spatial>, field: &Field) -> f32 {
-		field.distance(space, vec3a(0.0, 0.0, 0.0))
+		field.distance(space, self.origin.into())
 	}
-	fn serialize(
-		&self,
-		_distance_link: &DistanceLink,
-		local_to_handler_matrix: Mat4,
-	) -> InputDataType {
+	fn transform(&mut self, method: &InputMethod, handler: &InputHandler) {
+		let local_to_handler_matrix =
+			Spatial::space_to_space_matrix(Some(&method.spatial), Some(&handler.spatial))
+				* Mat4::from_rotation_translation(self.orientation.into(), self.origin.into());
 		let (_, orientation, origin) = local_to_handler_matrix.to_scale_rotation_translation();
-		InputDataType::Tip(FlatTip {
-			origin: origin.into(),
-			orientation: orientation.into(),
-			radius: self.radius,
-		})
+		self.origin = origin.into();
+		self.orientation = orientation.into();
 	}
 }
-
-pub fn create_tip_flex(_node: &Node, calling_client: Arc<Client>, data: &[u8]) -> Result<()> {
-	#[derive(Deserialize)]
-	struct CreateTipInfo<'a> {
-		name: &'a str,
-		parent_path: &'a str,
-		transform: Transform,
-		radius: f32,
-		datamap: Option<Vec<u8>>,
-	}
-	let info: CreateTipInfo = deserialize(data)?;
-	let node = Node::create(&calling_client, "/input/method/tip", info.name, true);
-	let parent = find_spatial_parent(&calling_client, info.parent_path)?;
-	let transform = parse_transform(info.transform, true, true, false)?;
-
-	let node = node.add_to_scenegraph();
-	Spatial::add_to(&node, Some(parent), transform)?;
-	InputMethod::add_to(
-		&node,
-		InputType::Tip(Tip {
-			radius: info.radius,
-		}),
-		info.datamap.and_then(|datamap| Datamap::new(datamap).ok()),
-	)?;
-	node.add_local_signal("setRadius", Tip::set_radius);
-	Ok(())
-}

src/nodes/items/camera.rs

@@ -0,0 +1,187 @@
+#![allow(dead_code)]
+use super::{Item, ItemType, create_item_acceptor_flex, register_item_ui_flex};
+use crate::bail;
+use crate::core::error::Result;
+use crate::nodes::Aspect;
+use crate::nodes::AspectIdentifier;
+use crate::nodes::items::ITEM_ACCEPTOR_ASPECT_ALIAS_INFO;
+use crate::nodes::items::ITEM_ASPECT_ALIAS_INFO;
+use crate::{
+	core::{client::Client, registry::Registry, scenegraph::MethodResponseSender},
+	nodes::{
+		Message, Node,
+		drawable::model::ModelPart,
+		items::TypeInfo,
+		spatial::{Spatial, Transform},
+	},
+};
+use glam::Mat4;
+use lazy_static::lazy_static;
+use mint::{ColumnMatrix4, Vector2};
+use parking_lot::Mutex;
+
+use stardust_xr::schemas::flex::{deserialize, serialize};
+use std::sync::Arc;
+
+stardust_xr_server_codegen::codegen_item_camera_protocol!();
+lazy_static! {
+	pub(super) static ref ITEM_TYPE_INFO_CAMERA: TypeInfo = TypeInfo {
+		type_name: "camera",
+		alias_info: CAMERA_ITEM_ASPECT_ALIAS_INFO.clone(),
+		ui_node_id: INTERFACE_NODE_ID,
+		ui: Default::default(),
+		items: Registry::new(),
+		acceptors: Registry::new(),
+		add_acceptor_aspect: |node| {
+			node.add_aspect(CameraItemAcceptor);
+		},
+		add_ui_aspect: |node| {
+			node.add_aspect(CameraItemUi);
+		},
+		new_acceptor_fn: |node, acceptor, acceptor_field| {
+			let _ = camera_item_ui_client::create_acceptor(node, acceptor, acceptor_field);
+		}
+	};
+}
+
+struct FrameInfo {
+	proj_matrix: Mat4,
+	px_size: Vector2<u32>,
+}
+
+pub struct CameraItem {
+	space: Arc<Spatial>,
+	frame_info: Mutex<FrameInfo>,
+	applied_to: Registry<ModelPart>,
+	apply_to: Registry<ModelPart>,
+}
+#[allow(unused)]
+impl CameraItem {
+	pub fn add_to(node: &Arc<Node>, proj_matrix: Mat4, px_size: Vector2<u32>) {
+		let item = Arc::new(CameraItem {
+			space: node.get_aspect::<Spatial>().unwrap().clone(),
+			frame_info: Mutex::new(FrameInfo {
+				proj_matrix,
+				px_size,
+			}),
+			applied_to: Registry::new(),
+			apply_to: Registry::new(),
+		});
+		Item::add_to(node, &ITEM_TYPE_INFO_CAMERA, ItemType::Camera(item.clone()));
+		node.add_aspect_raw(item);
+	}
+
+	fn frame_flex(
+		node: Arc<Node>,
+		_calling_client: Arc<Client>,
+		_message: Message,
+		response: MethodResponseSender,
+	) {
+		response.wrap_sync(move || {
+			let ItemType::Camera(_camera) = &node.get_aspect::<Item>().unwrap().specialization
+			else {
+				bail!("Wrong item type?");
+			};
+			Ok(serialize(())?.into())
+		});
+	}
+
+	fn apply_preview_material_flex(
+		node: Arc<Node>,
+		calling_client: Arc<Client>,
+		message: Message,
+	) -> Result<()> {
+		let ItemType::Camera(camera) = &node.get_aspect::<Item>().unwrap().specialization else {
+			bail!("Wrong item type?");
+		};
+		let model_part_node =
+			calling_client.get_node("Model part", deserialize(&message.data).unwrap())?;
+		let model_part = model_part_node.get_aspect::<ModelPart>()?;
+		camera.applied_to.add_raw(&model_part);
+		camera.apply_to.add_raw(&model_part);
+		Ok(())
+	}
+
+	pub fn send_ui_item_created(&self, node: &Node, item: &Arc<Node>) {
+		let _ = camera_item_ui_client::create_item(node, item);
+	}
+	pub fn send_acceptor_item_created(&self, node: &Node, item: &Arc<Node>) {
+		let _ = camera_item_acceptor_client::capture_item(node, item);
+	}
+}
+impl AspectIdentifier for CameraItem {
+	impl_aspect_for_camera_item_aspect_id! {}
+}
+impl Aspect for CameraItem {
+	impl_aspect_for_camera_item_aspect! {}
+}
+impl CameraItemAspect for CameraItem {}
+
+pub struct CameraItemUi;
+impl AspectIdentifier for CameraItemUi {
+	impl_aspect_for_camera_item_ui_aspect_id! {}
+}
+impl Aspect for CameraItemUi {
+	impl_aspect_for_camera_item_ui_aspect! {}
+}
+impl CameraItemUiAspect for CameraItemUi {}
+
+pub struct CameraItemAcceptor;
+impl AspectIdentifier for CameraItemAcceptor {
+	impl_aspect_for_camera_item_acceptor_aspect_id! {}
+}
+impl Aspect for CameraItemAcceptor {
+	impl_aspect_for_camera_item_acceptor_aspect! {}
+}
+impl CameraItemAcceptorAspect for CameraItemAcceptor {
+	fn capture_item(node: Arc<Node>, _calling_client: Arc<Client>, item: Arc<Node>) -> Result<()> {
+		super::acceptor_capture_item_flex(node, item)
+	}
+}
+
+impl InterfaceAspect for Interface {
+	#[doc = "Create a camera item at a specific location"]
+	fn create_camera_item(
+		_node: Arc<Node>,
+		calling_client: Arc<Client>,
+		id: u64,
+		parent: Arc<Node>,
+		transform: Transform,
+		proj_matrix: ColumnMatrix4<f32>,
+		px_size: Vector2<u32>,
+	) -> Result<()> {
+		let space = parent.get_aspect::<Spatial>()?;
+		let transform = transform.to_mat4(true, true, false);
+
+		let node = Node::from_id(&calling_client, id, false).add_to_scenegraph()?;
+		Spatial::add_to(&node, None, transform * space.global_transform(), false);
+		CameraItem::add_to(&node, proj_matrix.into(), px_size);
+		Ok(())
+	}
+
+	#[doc = "Register this client to manage camera items and create default 3D UI for them."]
+	fn register_camera_item_ui(node: Arc<Node>, calling_client: Arc<Client>) -> Result<()> {
+		node.add_aspect(CameraItemUi);
+		register_item_ui_flex(calling_client, &ITEM_TYPE_INFO_CAMERA)
+	}
+
+	#[doc = "Create an item acceptor to allow temporary ownership of a given type of item. Creates a node at `/item/camera/acceptor/<name>`."]
+	fn create_camera_item_acceptor(
+		node: Arc<Node>,
+		calling_client: Arc<Client>,
+		id: u64,
+		parent: Arc<Node>,
+		transform: Transform,
+		field: Arc<Node>,
+	) -> Result<()> {
+		node.add_aspect(CameraItemAcceptor);
+		create_item_acceptor_flex(
+			calling_client,
+			id,
+			parent,
+			transform,
+			&ITEM_TYPE_INFO_CAMERA,
+			field,
+		)
+	}
+}

src/nodes/items/environment.rs

@@ -1,85 +0,0 @@
-use super::{Item, ItemSpecialization, ItemType, ITEM_TYPE_INFO_ENVIRONMENT};
-use crate::{
-	core::client::{Client, INTERNAL_CLIENT},
-	nodes::{
-		spatial::{find_spatial_parent, parse_transform, Spatial},
-		Node,
-	},
-};
-use anyhow::{anyhow, Result};
-use serde::Deserialize;
-use stardust_xr::{
-	schemas::flex::{deserialize, serialize},
-	values::Transform,
-};
-use std::sync::Arc;
-
-pub struct EnvironmentItem {
-	path: String,
-}
-impl EnvironmentItem {
-	pub fn add_to(node: &Arc<Node>, path: String) {
-		Item::add_to(
-			node,
-			&ITEM_TYPE_INFO_ENVIRONMENT,
-			ItemType::Environment(EnvironmentItem { path }),
-		);
-		node.add_local_method("getPath", EnvironmentItem::get_path_flex);
-	}
-
-	fn get_path_flex(node: &Node, _calling_client: Arc<Client>, _data: &[u8]) -> Result<Vec<u8>> {
-		let path: Result<String> = match &node.item.get().unwrap().specialization {
-			ItemType::Environment(env) => Ok(env.path.clone()),
-			_ => Err(anyhow!("")),
-		};
-		Ok(flexbuffers::singleton(path?.as_str()))
-	}
-}
-impl ItemSpecialization for EnvironmentItem {
-	fn serialize_start_data(&self, id: &str) -> Vec<u8> {
-		serialize((id, self.path.as_str())).unwrap()
-	}
-}
-
-pub(super) fn create_environment_item_flex(
-	_node: &Node,
-	calling_client: Arc<Client>,
-	data: &[u8],
-) -> Result<()> {
-	#[derive(Deserialize)]
-	struct CreateEnvironmentItemInfo<'a> {
-		name: &'a str,
-		parent_path: &'a str,
-		transform: Transform,
-		item_data: String,
-	}
-	let info: CreateEnvironmentItemInfo = deserialize(data)?;
-	let parent_name = format!("/item/{}/item/", ITEM_TYPE_INFO_ENVIRONMENT.type_name);
-	let node = Node::create(&INTERNAL_CLIENT, &parent_name, info.name, true);
-	let space = find_spatial_parent(&calling_client, info.parent_path)?;
-	let transform = parse_transform(info.transform, true, true, false)?;
-	let node = node.add_to_scenegraph();
-	Spatial::add_to(&node, None, transform * space.global_transform())?;
-	EnvironmentItem::add_to(&node, info.item_data);
-	node.item
-		.get()
-		.unwrap()
-		.make_alias(&calling_client, &parent_name);
-	Ok(())
-}
-
-pub(super) fn create_environment_item_acceptor_flex(
-	_node: &Node,
-	calling_client: Arc<Client>,
-	data: &[u8],
-) -> Result<()> {
-	super::create_item_acceptor_flex(calling_client, data, &ITEM_TYPE_INFO_ENVIRONMENT)
-}
-
-pub(super) fn register_environment_item_ui_flex(
-	_node: &Node,
-	calling_client: Arc<Client>,
-	_data: &[u8],
-) -> Result<()> {
-	super::register_item_ui_flex(calling_client, &ITEM_TYPE_INFO_ENVIRONMENT)
-}

src/nodes/items/mod.rs

@@ -1,100 +1,71 @@
-mod environment;
+pub mod camera;
+pub mod panel;
 
-use self::environment::EnvironmentItem;
-use super::fields::Field;
-use super::spatial::{find_spatial_parent, parse_transform, Spatial};
-use super::{Alias, Node};
-use crate::core::client::{Client, INTERNAL_CLIENT};
-use crate::core::nodelist::LifeLinkedNodeList;
+use self::camera::CameraItem;
+use self::panel::PanelItemTrait;
+use super::alias::AliasList;
+use super::fields::{FIELD_ALIAS_INFO, Field};
+use super::spatial::Spatial;
+use super::{Alias, Aspect, AspectIdentifier, Node};
+use crate::core::client::Client;
+use crate::core::error::Result;
 use crate::core::registry::Registry;
+use crate::ensure;
 use crate::nodes::alias::AliasInfo;
-use crate::nodes::fields::find_field;
-use crate::wayland::panel_item::{register_panel_item_ui_flex, PanelItem};
-use anyhow::{anyhow, ensure, Result};
-use lazy_static::lazy_static;
-use nanoid::nanoid;
+use crate::nodes::spatial::SPATIAL_ASPECT_ALIAS_INFO;
+use crate::nodes::spatial::Transform;
 use parking_lot::Mutex;
-use serde::Deserialize;
-use stardust_xr::schemas::flex::deserialize;
-use stardust_xr::values::Transform;
-
-use std::ops::Deref;
+use std::hash::Hash;
 use std::sync::{Arc, Weak};
 
-lazy_static! {
-	static ref ITEM_ALIAS_LOCAL_SIGNALS: Vec<&'static str> = vec![
-		"getTransform",
-		"setTransform",
-		"setSpatialParent",
-		"setSpatialParentInPlace",
-		"setZoneable",
-		"release",
-	];
-	static ref ITEM_ALIAS_LOCAL_METHODS: Vec<&'static str> = vec!["captureInto"];
-	static ref ITEM_ALIAS_REMOTE_SIGNALS: Vec<&'static str> = vec![];
-	static ref ITEM_ALIAS_REMOTE_METHODS: Vec<&'static str> = vec![];
-	static ref ITEM_TYPE_INFO_ENVIRONMENT: TypeInfo = TypeInfo {
-		type_name: "environment",
-		aliased_local_signals: vec!["applySkyTex", "applySkyLight"],
-		aliased_local_methods: vec![],
-		aliased_remote_signals: vec![],
-		aliased_remote_methods: vec![],
-		ui: Default::default(),
-		items: Registry::new(),
-		acceptors: Registry::new(),
-	};
-}
+stardust_xr_server_codegen::codegen_item_protocol!();
 
 fn capture(item: &Arc<Item>, acceptor: &Arc<ItemAcceptor>) {
-	if item.captured_acceptor.lock().upgrade().is_some() {
+	if item.captured_acceptor.lock().strong_count() > 0 {
 		release(item);
 	}
 	*item.captured_acceptor.lock() = Arc::downgrade(acceptor);
 	acceptor.handle_capture(item);
 	if let Some(ui) = item.type_info.ui.lock().upgrade() {
-		ui.handle_capture(item);
+		ui.handle_capture_item(item, acceptor);
 	}
 }
-fn release(item: &Arc<Item>) {
-	if let Some(acceptor) = item.captured_acceptor.lock().upgrade() {
-		*item.captured_acceptor.lock() = Weak::default();
+fn release(item: &Item) {
+	let mut captured_acceptor = item.captured_acceptor.lock();
+	if let Some(acceptor) = captured_acceptor.upgrade().as_ref() {
+		*captured_acceptor = Weak::default();
 		acceptor.handle_release(item);
 		if let Some(ui) = item.type_info.ui.lock().upgrade() {
-			ui.handle_release(item);
+			ui.handle_release_item(item, acceptor);
 		}
 	}
 }
 
 pub struct TypeInfo {
 	pub type_name: &'static str,
-	pub aliased_local_signals: Vec<&'static str>,
-	pub aliased_local_methods: Vec<&'static str>,
-	pub aliased_remote_signals: Vec<&'static str>,
-	pub aliased_remote_methods: Vec<&'static str>,
+	pub alias_info: AliasInfo,
+	pub ui_node_id: u64,
 	pub ui: Mutex<Weak<ItemUI>>,
 	pub items: Registry<Item>,
 	pub acceptors: Registry<ItemAcceptor>,
+	pub add_ui_aspect: fn(node: &Node),
+	pub add_acceptor_aspect: fn(node: &Node),
+	pub new_acceptor_fn: fn(node: &Node, acceptor: &Arc<Node>, acceptor_field: &Arc<Node>),
 }
-
-fn capture_into_flex(node: &Node, calling_client: Arc<Client>, data: &[u8]) -> Result<()> {
-	let acceptor_path = flexbuffers::Reader::get_root(data)?
-		.get_str()
-		.map_err(|_| anyhow!("Acceptor path is not a string"))?;
-	let acceptor = calling_client
-		.scenegraph
-		.get_node(acceptor_path)
-		.ok_or_else(|| anyhow!("Acceptor node not found"))?;
-	let acceptor = acceptor
-		.item_acceptor
-		.get()
-		.ok_or_else(|| anyhow!("Acceptor node is not an acceptor!"))?;
-	capture(node.item.get().unwrap(), acceptor);
-	Ok(())
+impl Hash for TypeInfo {
+	fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
+		self.type_name.hash(state);
+	}
 }
+impl PartialEq for TypeInfo {
+	fn eq(&self, other: &Self) -> bool {
+		self.type_name == other.type_name
+	}
+}
+impl Eq for TypeInfo {}
 
 pub struct Item {
-	node: Weak<Node>,
-	uid: String,
+	spatial: Arc<Spatial>,
 	type_info: &'static TypeInfo,
 	captured_acceptor: Mutex<Weak<ItemAcceptor>>,
 	pub specialization: ItemType,
@@ -105,81 +76,99 @@ impl Item {
 		type_info: &'static TypeInfo,
 		specialization: ItemType,
 	) -> Arc<Self> {
-		node.add_local_signal("captureInto", capture_into_flex);
 		let item = Item {
-			node: Arc::downgrade(node),
-			uid: nanoid!(),
+			spatial: node.aspects.get::<Spatial>().unwrap(),
 			type_info,
 			captured_acceptor: Default::default(),
 			specialization,
 		};
 		let item = type_info.items.add(item);
+
 		if let Some(ui) = type_info.ui.lock().upgrade() {
 			ui.handle_create_item(&item);
 		}
-		let _ = node.item.set(item.clone());
+		node.add_aspect_raw(item.clone());
+
+		// if let Some(auto_acceptor) = node.get_client().and_then(|client| {
+		// 	client
+		// 		.state
+		// 		.as_ref()
+		// 		.and_then(|settings| settings.acceptors.get(type_info))
+		// 		.and_then(|acceptor| acceptor.upgrade())
+		// }) {
+		// 	capture(&item, &auto_acceptor);
+		// }
+
 		item
 	}
-	fn make_alias(&self, client: &Arc<Client>, parent: &str) -> (Arc<Node>, Arc<Alias>) {
-		let node = Alias::new(
+	fn make_alias(&self, client: &Arc<Client>, alias_list: &AliasList) -> Result<Arc<Node>> {
+		Alias::create(
+			&self.spatial.node().unwrap(),
 			client,
-			parent,
-			&self.uid,
-			&self.node.upgrade().unwrap(),
-			AliasInfo {
-				local_signals: [
-					&self.type_info.aliased_local_signals,
-					ITEM_ALIAS_LOCAL_SIGNALS.as_slice(),
-				]
-				.concat(),
-				local_methods: [
-					&self.type_info.aliased_local_methods,
-					ITEM_ALIAS_LOCAL_METHODS.as_slice(),
-				]
-				.concat(),
-				remote_signals: [
-					&self.type_info.aliased_remote_signals,
-					ITEM_ALIAS_REMOTE_SIGNALS.as_slice(),
-				]
-				.concat(),
-			},
-		);
-		let alias = node.alias.get().unwrap().clone();
-		(node, alias)
+			self.type_info.alias_info.clone() + ITEM_ASPECT_ALIAS_INFO.clone(),
+			Some(alias_list),
+		)
+	}
+}
+impl AspectIdentifier for Item {
+	impl_aspect_for_item_aspect_id! {}
+}
+impl Aspect for Item {
+	impl_aspect_for_item_aspect! {}
+}
+impl ItemAspect for Item {
+	fn release(node: Arc<Node>, _calling_client: Arc<Client>) -> Result<()> {
+		let item = node.get_aspect::<Item>()?;
+		release(&item);
+		Ok(())
 	}
 }
 impl Drop for Item {
 	fn drop(&mut self) {
 		self.type_info.items.remove(self);
+		release(self);
 		if let Some(ui) = self.type_info.ui.lock().upgrade() {
 			ui.handle_destroy_item(self);
 		}
 	}
 }
 
-pub trait ItemSpecialization {
-	fn serialize_start_data(&self, id: &str) -> Vec<u8>;
-}
-
+#[cfg_attr(not(feature = "wayland"), allow(dead_code))]
 pub enum ItemType {
-	Environment(EnvironmentItem),
-	Panel(PanelItem),
+	Camera(Arc<CameraItem>),
+	Panel(Arc<dyn PanelItemTrait>),
 }
-impl Deref for ItemType {
-	type Target = dyn ItemSpecialization;
-
-	fn deref(&self) -> &Self::Target {
+impl ItemType {
+	fn send_ui_item_created(&self, node: &Node, item: &Arc<Node>) {
 		match self {
-			ItemType::Environment(item) => item,
-			ItemType::Panel(item) => item,
+			ItemType::Camera(c) => c.send_ui_item_created(node, item),
+			ItemType::Panel(p) => p.send_ui_item_created(node, item),
+		}
+	}
+	fn send_acceptor_item_created(&self, node: &Node, item: &Arc<Node>) {
+		match self {
+			ItemType::Camera(c) => c.send_acceptor_item_created(node, item),
+			ItemType::Panel(p) => p.send_acceptor_item_created(node, item),
 		}
 	}
 }
+// impl Deref for ItemType {
+// 	type Target = dyn ItemSpecialization;
+
+// 	fn deref(&self) -> &Self::Target {
+// 		match self {
+// 			ItemType::Environment(item) => item,
+// 			ItemType::Panel(item) => item.as_ref(),
+// 		}
+// 	}
+// }
 
 pub struct ItemUI {
 	node: Weak<Node>,
 	type_info: &'static TypeInfo,
-	aliases: LifeLinkedNodeList,
+	item_aliases: AliasList,
+	acceptor_aliases: AliasList,
+	acceptor_field_aliases: AliasList,
 }
 impl ItemUI {
 	fn add_to(node: &Arc<Node>, type_info: &'static TypeInfo) -> Result<()> {
@@ -191,59 +180,121 @@ impl ItemUI {
 		let ui = Arc::new(ItemUI {
 			node: Arc::downgrade(node),
 			type_info,
-			aliases: Default::default(),
+			item_aliases: AliasList::default(),
+			acceptor_aliases: AliasList::default(),
+			acceptor_field_aliases: AliasList::default(),
 		});
 		*type_info.ui.lock() = Arc::downgrade(&ui);
-		let _ = node.item_ui.set(ui.clone());
+		node.add_aspect_raw(ui.clone());
 
 		for item in type_info.items.get_valid_contents() {
 			ui.handle_create_item(&item);
 		}
+		for acceptor in type_info.acceptors.get_valid_contents() {
+			ui.handle_create_acceptor(&acceptor);
+		}
 		Ok(())
 	}
-	fn send_state(&self, state: &str, name: &str) {
-		let _ = self
-			.node
-			.upgrade()
-			.unwrap()
-			.send_remote_signal(state, flexbuffers::singleton(name).as_slice());
-	}
 
 	fn handle_create_item(&self, item: &Item) {
-		let node = self.node.upgrade().unwrap();
-		if let Some(client) = node.get_client() {
-			let (alias_node, _) =
-				item.make_alias(&client, &(node.get_path().to_string() + "/item"));
-			self.aliases.add(Arc::downgrade(&alias_node));
+		let Some(node) = self.node.upgrade() else {
+			return;
+		};
+		let Some(client) = node.get_client() else {
+			return;
+		};
 
-			let _ = node.send_remote_signal(
-				"create",
-				&item.specialization.serialize_start_data(&item.uid),
-			);
-		}
+		let Ok(item_alias) = item.make_alias(&client, &self.item_aliases) else {
+			return;
+		};
+
+		item.specialization.send_ui_item_created(&node, &item_alias);
+	}
+	fn handle_capture_item(&self, item: &Item, acceptor: &ItemAcceptor) {
+		let Some(item_alias) = self.item_aliases.get_from_aspect(item) else {
+			return;
+		};
+		let Some(acceptor_alias) = self.acceptor_aliases.get_from_aspect(acceptor) else {
+			return;
+		};
+		let _ = item_ui_client::capture_item(
+			&self.node.upgrade().unwrap(),
+			item_alias.id,
+			acceptor_alias.id,
+		);
+	}
+	fn handle_release_item(&self, item: &Item, acceptor: &ItemAcceptor) {
+		let Some(item_alias) = self.item_aliases.get_from_aspect(item) else {
+			return;
+		};
+		let Some(acceptor_alias) = self.acceptor_aliases.get_from_aspect(acceptor) else {
+			return;
+		};
+		let _ = item_ui_client::release_item(
+			&self.node.upgrade().unwrap(),
+			item_alias.id,
+			acceptor_alias.id,
+		);
 	}
 	fn handle_destroy_item(&self, item: &Item) {
-		self.send_state("destroy", item.uid.as_str());
-	}
-	fn handle_capture(&self, item: &Item) {
-		self.send_state("capture", item.uid.as_str());
-	}
-	fn handle_release(&self, item: &Item) {
-		self.send_state("release", item.uid.as_str());
+		let Some(item_alias) = self
+			.item_aliases
+			.get_from_original_node(item.spatial.node.clone())
+		else {
+			return;
+		};
+		let _ = item_ui_client::destroy_item(&self.node.upgrade().unwrap(), item_alias.id);
+		self.item_aliases.remove_aspect(item);
 	}
 	fn handle_create_acceptor(&self, acceptor: &ItemAcceptor) {
-		let node = self.node.upgrade().unwrap();
-		if let Some(client) = node.get_client() {
-			let aliases = acceptor.make_aliases(
-				&client,
-				&format!("/item/{}/acceptor", self.type_info.type_name),
-			);
-			aliases
-				.iter()
-				.for_each(|alias| self.aliases.add(Arc::downgrade(alias)));
-		}
+		let Some(node) = self.node.upgrade() else {
+			return;
+		};
+		let Some(client) = node.get_client() else {
+			return;
+		};
+
+		let Some(acceptor_node) = acceptor.spatial.node() else {
+			return;
+		};
+		let Ok(acceptor_alias) = Alias::create(
+			&acceptor_node,
+			&client,
+			ITEM_ACCEPTOR_ASPECT_ALIAS_INFO.clone(),
+			Some(&self.acceptor_aliases),
+		) else {
+			return;
+		};
+
+		let Some(acceptor_field_node) = acceptor.field.spatial.node() else {
+			return;
+		};
+		let Ok(acceptor_field_alias) = Alias::create(
+			&acceptor_field_node,
+			&client,
+			FIELD_ALIAS_INFO.clone(),
+			Some(&self.acceptor_aliases),
+		) else {
+			return;
+		};
+
+		(acceptor.type_info.new_acceptor_fn)(&node, &acceptor_alias, &acceptor_field_alias);
 	}
-	fn handle_destroy_acceptor(&self, _acceptor: &ItemAcceptor) {}
+	fn handle_destroy_acceptor(&self, acceptor: &ItemAcceptor) {
+		let acceptor_alias = self.acceptor_aliases.get_from_aspect(acceptor).unwrap();
+		let _ = item_ui_client::destroy_acceptor(&self.node.upgrade().unwrap(), acceptor_alias.id);
+
+		self.acceptor_aliases
+			.remove_aspect(acceptor.spatial.as_ref());
+		self.acceptor_field_aliases
+			.remove_aspect(acceptor.field.as_ref());
+	}
+}
+impl AspectIdentifier for ItemUI {
+	impl_aspect_for_item_ui_aspect_id! {}
+}
+impl Aspect for ItemUI {
+	impl_aspect_for_item_ui_aspect! {}
 }
 impl Drop for ItemUI {
 	fn drop(&mut self) {
@@ -252,127 +303,105 @@ impl Drop for ItemUI {
 }
 
 pub struct ItemAcceptor {
-	node: Weak<Node>,
-	type_info: &'static TypeInfo,
-	field: Mutex<Weak<Field>>,
-	accepted: Registry<Item>,
+	spatial: Arc<Spatial>,
+	pub type_info: &'static TypeInfo,
+	field: Arc<Field>,
+	accepted_aliases: AliasList,
+	accepted_registry: Registry<Item>,
 }
 impl ItemAcceptor {
-	fn add_to(node: &Arc<Node>, type_info: &'static TypeInfo, field: Weak<Field>) {
+	fn add_to(node: &Arc<Node>, type_info: &'static TypeInfo, field: Arc<Field>) {
 		let acceptor = type_info.acceptors.add(ItemAcceptor {
-			node: Arc::downgrade(node),
+			spatial: node.get_aspect::<Spatial>().unwrap(),
 			type_info,
-			field: Mutex::new(field),
-			accepted: Registry::new(),
+			field,
+			accepted_aliases: AliasList::default(),
+			accepted_registry: Registry::new(),
 		});
 		if let Some(ui) = type_info.ui.lock().upgrade() {
 			ui.handle_create_acceptor(&acceptor);
 		}
-		let _ = node.item_acceptor.set(acceptor);
+		node.add_aspect_raw(acceptor.clone());
 	}
-	fn make_aliases(&self, client: &Arc<Client>, parent: &str) -> Vec<Arc<Node>> {
-		let mut aliases = Vec::new();
-		let acceptor_node = &self.node.upgrade().unwrap();
-		let acceptor_alias = Alias::new(
-			client,
-			parent,
-			acceptor_node.uid.as_str(),
-			acceptor_node,
-			AliasInfo {
-				local_signals: vec!["release"],
-				..Default::default()
-			},
-		);
-		if let Some(field) = self.field.lock().upgrade() {
-			let acceptor_field_alias = Alias::new(
-				client,
-				acceptor_alias.get_path(),
-				"field",
-				&field.spatial_ref().node.upgrade().unwrap(),
-				AliasInfo::default(),
-			);
 
-			aliases.push(acceptor_field_alias);
-		}
-		aliases.push(acceptor_alias);
-		aliases
-	}
-	fn send_event(&self, state: &str, name: &str) {
-		let _ = self
-			.node
-			.upgrade()
-			.unwrap()
-			.send_remote_signal(state, flexbuffers::singleton(name).as_slice());
-	}
 	fn handle_capture(&self, item: &Arc<Item>) {
-		self.accepted.add_raw(item);
-		self.send_event("capture", item.uid.as_str());
+		let Some(node) = self.spatial.node() else {
+			return;
+		};
+		let Some(client) = node.get_client() else {
+			return;
+		};
+
+		self.accepted_registry.add_raw(item);
+		let Ok(alias_node) = item.make_alias(&client, &self.accepted_aliases) else {
+			return;
+		};
+
+		item.specialization
+			.send_acceptor_item_created(&node, &alias_node);
 	}
 	fn handle_release(&self, item: &Item) {
-		self.accepted.remove(item);
-		self.send_event("release", item.uid.as_str());
+		self.accepted_registry.remove(item);
+		self.accepted_aliases.remove_aspect(item);
+
+		let Some(node) = self.spatial.node() else {
+			return;
+		};
+		let alias = self.accepted_aliases.get_from_aspect(item).unwrap();
+		let _ = item_acceptor_client::release_item(&node, alias.id);
 	}
 }
+impl AspectIdentifier for ItemAcceptor {
+	impl_aspect_for_item_acceptor_aspect_id! {}
+}
+impl Aspect for ItemAcceptor {
+	impl_aspect_for_item_acceptor_aspect! {}
+}
+impl ItemAcceptorAspect for ItemAcceptor {}
 impl Drop for ItemAcceptor {
 	fn drop(&mut self) {
 		self.type_info.acceptors.remove(self);
+		for item in self.accepted_registry.get_valid_contents() {
+			release(&item);
+		}
 		if let Some(ui) = self.type_info.ui.lock().upgrade() {
 			ui.handle_destroy_acceptor(self);
 		}
 	}
 }
 
-pub fn create_interface(client: &Arc<Client>) {
-	let node = Node::create(client, "", "item", false);
-	node.add_local_signal(
-		"createEnvironmentItem",
-		environment::create_environment_item_flex,
-	);
-	node.add_local_signal(
-		"registerEnvironmentItemUI",
-		environment::register_environment_item_ui_flex,
-	);
-	node.add_local_signal("registerPanelItemUI", register_panel_item_ui_flex);
-	node.add_local_signal(
-		"createEnvironmentItemAcceptor",
-		environment::create_environment_item_acceptor_flex,
-	);
-	node.add_to_scenegraph();
-}
-
-pub(self) fn create_item_acceptor_flex(
-	calling_client: Arc<Client>,
-	data: &[u8],
-	type_info: &'static TypeInfo,
-) -> Result<()> {
-	#[derive(Deserialize)]
-	struct CreateItemAcceptorInfo<'a> {
-		name: &'a str,
-		parent_path: &'a str,
-		transform: Transform,
-		field_path: &'a str,
-	}
-	let info: CreateItemAcceptorInfo = deserialize(data)?;
-	let parent_name = format!("/item/{}/acceptor/", ITEM_TYPE_INFO_ENVIRONMENT.type_name);
-	let space = find_spatial_parent(&calling_client, info.parent_path)?;
-	let transform = parse_transform(info.transform, true, true, false)?;
-	let field = find_field(&calling_client, info.field_path)?;
-
-	let node = Node::create(&INTERNAL_CLIENT, &parent_name, info.name, true).add_to_scenegraph();
-	Spatial::add_to(&node, Some(space), transform)?;
-	ItemAcceptor::add_to(&node, type_info, Arc::downgrade(&field));
-	node.item
-		.get()
-		.unwrap()
-		.make_alias(&calling_client, &parent_name);
-	Ok(())
-}
-
 pub fn register_item_ui_flex(
 	calling_client: Arc<Client>,
 	type_info: &'static TypeInfo,
 ) -> Result<()> {
-	let ui = Node::create(&calling_client, "/item", type_info.type_name, true).add_to_scenegraph();
+	let ui = Node::from_id(&calling_client, type_info.ui_node_id, true).add_to_scenegraph()?;
 	ItemUI::add_to(&ui, type_info)?;
+	(type_info.add_ui_aspect)(&ui);
+	Ok(())
+}
+fn create_item_acceptor_flex(
+	calling_client: Arc<Client>,
+	id: u64,
+	parent: Arc<Node>,
+	transform: Transform,
+	type_info: &'static TypeInfo,
+	field: Arc<Node>,
+) -> Result<()> {
+	let space = parent.get_aspect::<Spatial>()?;
+	let field = field.get_aspect::<Field>()?;
+	let transform = transform.to_mat4(true, true, false);
+
+	let node = Node::from_id(&calling_client, id, true).add_to_scenegraph()?;
+	Spatial::add_to(&node, Some(space.clone()), transform, false);
+	ItemAcceptor::add_to(&node, type_info, field);
+	(type_info.add_acceptor_aspect)(&node);
+	Ok(())
+}
+
+fn acceptor_capture_item_flex(node: Arc<Node>, item: Arc<Node>) -> Result<()> {
+	let acceptor = node.get_aspect::<ItemAcceptor>()?;
+	let item = item.get_aspect::<Item>()?;
+	capture(&item, &acceptor);
+
 	Ok(())
 }

src/nodes/items/panel.rs

@@ -0,0 +1,524 @@
+use super::camera::CameraItemAcceptor;
+use super::{create_item_acceptor_flex, register_item_ui_flex};
+use crate::bail;
+use crate::core::error::Result;
+use crate::nodes::items::ITEM_ACCEPTOR_ASPECT_ALIAS_INFO;
+use crate::nodes::items::ITEM_ASPECT_ALIAS_INFO;
+use crate::nodes::{Aspect, AspectIdentifier};
+use crate::{
+	core::{
+		client::{Client, INTERNAL_CLIENT, get_env, state},
+		registry::Registry,
+	},
+	nodes::{
+		Node,
+		drawable::model::ModelPart,
+		items::{Item, ItemType, TypeInfo},
+		spatial::{Spatial, Transform},
+	},
+};
+use glam::Mat4;
+use lazy_static::lazy_static;
+use mint::Vector2;
+use parking_lot::Mutex;
+use slotmap::{DefaultKey, Key, KeyData, SlotMap};
+use std::sync::{Arc, Weak};
+use tracing::debug;
+
+stardust_xr_server_codegen::codegen_item_panel_protocol!();
+impl Default for Geometry {
+	fn default() -> Self {
+		Geometry {
+			origin: [0, 0].into(),
+			size: [0, 0].into(),
+		}
+	}
+}
+impl Copy for Geometry {}
+
+lazy_static! {
+	pub static ref KEYMAPS: Mutex<SlotMap<DefaultKey, String>> = Mutex::new(SlotMap::default());
+	pub static ref ITEM_TYPE_INFO_PANEL: TypeInfo = TypeInfo {
+		type_name: "panel",
+		alias_info: PANEL_ITEM_ASPECT_ALIAS_INFO.clone(),
+		ui_node_id: INTERFACE_NODE_ID,
+		ui: Default::default(),
+		items: Registry::new(),
+		acceptors: Registry::new(),
+		add_acceptor_aspect: |node| {
+			node.add_aspect(PanelItemUi);
+		},
+		add_ui_aspect: |node| {
+			node.add_aspect(PanelItemAcceptor);
+		},
+		new_acceptor_fn: |node, acceptor, acceptor_field| {
+			let _ = panel_item_ui_client::create_acceptor(node, acceptor, acceptor_field);
+		}
+	};
+}
+
+pub trait Backend: Send + Sync + 'static {
+	fn start_data(&self) -> Result<PanelItemInitData>;
+
+	fn apply_cursor_material(&self, model_part: &Arc<ModelPart>);
+	fn apply_surface_material(&self, surface: SurfaceId, model_part: &Arc<ModelPart>);
+
+	fn close_toplevel(&self);
+	fn auto_size_toplevel(&self);
+	fn set_toplevel_size(&self, size: Vector2<u32>);
+	fn set_toplevel_focused_visuals(&self, focused: bool);
+
+	fn pointer_motion(&self, surface: &SurfaceId, position: Vector2<f32>);
+	fn pointer_button(&self, surface: &SurfaceId, button: u32, pressed: bool);
+	fn pointer_scroll(
+		&self,
+		surface: &SurfaceId,
+		scroll_distance: Option<Vector2<f32>>,
+		scroll_steps: Option<Vector2<f32>>,
+	);
+
+	fn keyboard_key(&self, surface: &SurfaceId, keymap_id: u64, key: u32, pressed: bool);
+
+	fn touch_down(&self, surface: &SurfaceId, id: u32, position: Vector2<f32>);
+	fn touch_move(&self, id: u32, position: Vector2<f32>);
+	fn touch_up(&self, id: u32);
+	fn reset_input(&self);
+}
+
+pub fn panel_item_from_node(node: &Node) -> Option<Arc<dyn PanelItemTrait>> {
+	let ItemType::Panel(panel_item) = &node.get_aspect::<Item>().ok()?.specialization else {
+		return None;
+	};
+	Some(panel_item.clone())
+}
+
+pub trait PanelItemTrait: Send + Sync + 'static {
+	fn backend(&self) -> &dyn Backend;
+	fn send_ui_item_created(&self, node: &Node, item: &Arc<Node>);
+	fn send_acceptor_item_created(&self, node: &Node, item: &Arc<Node>);
+}
+
+#[derive(Debug)]
+pub struct PanelItem<B: Backend> {
+	pub node: Weak<Node>,
+	pub backend: Box<B>,
+}
+impl<B: Backend> PanelItem<B> {
+	#[cfg_attr(not(feature = "wayland"), allow(dead_code))]
+	pub fn create(backend: Box<B>, pid: Option<i32>) -> (Arc<Node>, Arc<PanelItem<B>>) {
+		debug!(?pid, "Create panel item");
+
+		let startup_settings = pid
+			.and_then(|pid| get_env(pid).ok())
+			.and_then(|env| state(&env));
+
+		let node = Arc::new(Node::generate(&INTERNAL_CLIENT, true));
+		let spatial = Spatial::add_to(&node, None, Mat4::IDENTITY, false);
+		if let Some(startup_settings) = &startup_settings {
+			spatial.set_local_transform(startup_settings.root);
+		}
+
+		let panel_item = Arc::new(PanelItem {
+			node: Arc::downgrade(&node),
+			backend,
+		});
+
+		let generic_panel_item: Arc<dyn PanelItemTrait> = panel_item.clone();
+		Item::add_to(
+			&node,
+			&ITEM_TYPE_INFO_PANEL,
+			ItemType::Panel(generic_panel_item),
+		);
+		node.add_aspect_raw(panel_item.clone());
+
+		(node, panel_item)
+	}
+}
+
+// Remote signals
+#[allow(unused)]
+impl<B: Backend> PanelItem<B> {
+	pub fn toplevel_parent_changed(&self, parent: u64) {
+		let Some(node) = self.node.upgrade() else {
+			return;
+		};
+		panel_item_client::toplevel_parent_changed(&node, parent);
+	}
+	pub fn toplevel_title_changed(&self, title: &str) {
+		let Some(node) = self.node.upgrade() else {
+			return;
+		};
+		panel_item_client::toplevel_title_changed(&node, title);
+	}
+	pub fn toplevel_app_id_changed(&self, app_id: &str) {
+		let Some(node) = self.node.upgrade() else {
+			return;
+		};
+		panel_item_client::toplevel_app_id_changed(&node, app_id);
+	}
+	pub fn toplevel_fullscreen_active(&self, active: bool) {
+		let Some(node) = self.node.upgrade() else {
+			return;
+		};
+		panel_item_client::toplevel_fullscreen_active(&node, active);
+	}
+	pub fn toplevel_move_request(&self) {
+		let Some(node) = self.node.upgrade() else {
+			return;
+		};
+		panel_item_client::toplevel_move_request(&node);
+	}
+	pub fn toplevel_resize_request(&self, up: bool, down: bool, left: bool, right: bool) {
+		let Some(node) = self.node.upgrade() else {
+			return;
+		};
+		panel_item_client::toplevel_resize_request(&node, up, down, left, right);
+	}
+	pub fn toplevel_size_changed(&self, size: Vector2<u32>) {
+		let Some(node) = self.node.upgrade() else {
+			return;
+		};
+		panel_item_client::toplevel_size_changed(&node, size);
+	}
+
+	pub fn set_cursor(&self, geometry: Option<Geometry>) {
+		let Some(node) = self.node.upgrade() else {
+			return;
+		};
+		if let Some(geometry) = geometry {
+			panel_item_client::set_cursor(&node, &geometry);
+		} else {
+			panel_item_client::hide_cursor(&node);
+		}
+	}
+
+	pub fn create_child(&self, id: u64, info: &ChildInfo) {
+		let Some(node) = self.node.upgrade() else {
+			return;
+		};
+		panel_item_client::create_child(&node, id, info);
+	}
+	pub fn reposition_child(&self, id: u64, geometry: &Geometry) {
+		let Some(node) = self.node.upgrade() else {
+			return;
+		};
+		panel_item_client::reposition_child(&node, id, geometry);
+	}
+	pub fn destroy_child(&self, id: u64) {
+		let Some(node) = self.node.upgrade() else {
+			return;
+		};
+		panel_item_client::destroy_child(&node, id);
+	}
+}
+impl<B: Backend> AspectIdentifier for PanelItem<B> {
+	impl_aspect_for_panel_item_aspect_id! {}
+}
+impl<B: Backend> Aspect for PanelItem<B> {
+	impl_aspect_for_panel_item_aspect! {}
+}
+#[allow(unused)]
+impl<B: Backend> PanelItemAspect for PanelItem<B> {
+	#[doc = "Apply the cursor as a material to a model."]
+	fn apply_cursor_material(
+		node: Arc<Node>,
+		_calling_client: Arc<Client>,
+		model_part: Arc<Node>,
+	) -> Result<()> {
+		let Some(panel_item) = panel_item_from_node(&node) else {
+			return Ok(());
+		};
+		let model_part = model_part.get_aspect::<ModelPart>()?;
+
+		panel_item.backend().apply_cursor_material(&model_part);
+		Ok(())
+	}
+
+	#[doc = "Apply a surface's visuals as a material to a model."]
+	fn apply_surface_material(
+		node: Arc<Node>,
+		calling_client: Arc<Client>,
+		surface: SurfaceId,
+		model_part: Arc<Node>,
+	) -> Result<()> {
+		let Some(panel_item) = panel_item_from_node(&node) else {
+			return Ok(());
+		};
+		let model_part = model_part.get_aspect::<ModelPart>()?;
+
+		panel_item
+			.backend()
+			.apply_surface_material(surface, &model_part);
+		Ok(())
+	}
+
+	#[doc = "Try to close the toplevel.\n        \n        The panel item UI handler or panel item acceptor will drop the panel item if this succeeds."]
+	fn close_toplevel(node: Arc<Node>, _calling_client: Arc<Client>) -> Result<()> {
+		let Some(panel_item) = panel_item_from_node(&node) else {
+			return Ok(());
+		};
+		panel_item.backend().close_toplevel();
+		Ok(())
+	}
+
+	#[doc = "Request a resize of the surface to whatever size the 2D app wants."]
+	fn auto_size_toplevel(node: Arc<Node>, _calling_client: Arc<Client>) -> Result<()> {
+		let Some(panel_item) = panel_item_from_node(&node) else {
+			return Ok(());
+		};
+		panel_item.backend().auto_size_toplevel();
+		Ok(())
+	}
+
+	#[doc = "Request a resize of the surface (in pixels)."]
+	fn set_toplevel_size(
+		node: Arc<Node>,
+		_calling_client: Arc<Client>,
+		size: mint::Vector2<u32>,
+	) -> Result<()> {
+		let Some(panel_item) = panel_item_from_node(&node) else {
+			return Ok(());
+		};
+		panel_item.backend().set_toplevel_size(size);
+		Ok(())
+	}
+
+	#[doc = "Tell the toplevel to appear focused visually if true, or unfocused if false."]
+	fn set_toplevel_focused_visuals(
+		node: Arc<Node>,
+		_calling_client: Arc<Client>,
+		focused: bool,
+	) -> Result<()> {
+		let Some(panel_item) = panel_item_from_node(&node) else {
+			return Ok(());
+		};
+		panel_item.backend().set_toplevel_focused_visuals(focused);
+		Ok(())
+	}
+
+	#[doc = "Send an event to set the pointer's position (in pixels, relative to top-left of surface). This will activate the pointer."]
+	fn pointer_motion(
+		node: Arc<Node>,
+		_calling_client: Arc<Client>,
+		surface: SurfaceId,
+		position: mint::Vector2<f32>,
+	) -> Result<()> {
+		let Some(panel_item) = panel_item_from_node(&node) else {
+			return Ok(());
+		};
+		panel_item.backend().pointer_motion(&surface, position);
+		Ok(())
+	}
+
+	#[doc = "Send an event to set a pointer button's state if the pointer's active. The `button` is from the `input_event_codes` crate (e.g. BTN_LEFT for left click)."]
+	fn pointer_button(
+		node: Arc<Node>,
+		_calling_client: Arc<Client>,
+		surface: SurfaceId,
+		button: u32,
+		pressed: bool,
+	) -> Result<()> {
+		let Some(panel_item) = panel_item_from_node(&node) else {
+			return Ok(());
+		};
+		panel_item
+			.backend()
+			.pointer_button(&surface, button, pressed);
+		Ok(())
+	}
+
+	#[doc = "Send an event to scroll the pointer if it's active.\nScroll distance is a value in pixels corresponding to the `distance` the surface should be scrolled.\nScroll steps is a value in columns/rows corresponding to the wheel clicks of a mouse or such. This also supports fractions of a wheel click."]
+	fn pointer_scroll(
+		node: Arc<Node>,
+		_calling_client: Arc<Client>,
+		surface: SurfaceId,
+		scroll_distance: mint::Vector2<f32>,
+		scroll_steps: mint::Vector2<f32>,
+	) -> Result<()> {
+		let Some(panel_item) = panel_item_from_node(&node) else {
+			return Ok(());
+		};
+		panel_item
+			.backend()
+			.pointer_scroll(&surface, Some(scroll_distance), Some(scroll_steps));
+		Ok(())
+	}
+
+	#[doc = "Send an event to stop scrolling the pointer."]
+	fn pointer_stop_scroll(
+		node: Arc<Node>,
+		_calling_client: Arc<Client>,
+		surface: SurfaceId,
+	) -> Result<()> {
+		let Some(panel_item) = panel_item_from_node(&node) else {
+			return Ok(());
+		};
+		panel_item.backend().pointer_scroll(&surface, None, None);
+		Ok(())
+	}
+
+	#[doc = "Send a series of key presses and releases (positive keycode for pressed, negative for released)."]
+	fn keyboard_key(
+		node: Arc<Node>,
+		_calling_client: Arc<Client>,
+		surface: SurfaceId,
+		keymap_id: u64,
+		key: u32,
+		pressed: bool,
+	) -> Result<()> {
+		let Some(panel_item) = panel_item_from_node(&node) else {
+			return Ok(());
+		};
+		panel_item
+			.backend()
+			.keyboard_key(&surface, keymap_id, key, pressed);
+		Ok(())
+	}
+
+	#[doc = "Put a touch down on this surface with the unique ID `uid` at `position` (in pixels) from top left corner of the surface."]
+	fn touch_down(
+		node: Arc<Node>,
+		_calling_client: Arc<Client>,
+		surface: SurfaceId,
+		uid: u32,
+		position: mint::Vector2<f32>,
+	) -> Result<()> {
+		let Some(panel_item) = panel_item_from_node(&node) else {
+			return Ok(());
+		};
+		panel_item.backend().touch_down(&surface, uid, position);
+		Ok(())
+	}
+
+	#[doc = "Move an existing touch point."]
+	fn touch_move(
+		node: Arc<Node>,
+		_calling_client: Arc<Client>,
+		uid: u32,
+		position: mint::Vector2<f32>,
+	) -> Result<()> {
+		let Some(panel_item) = panel_item_from_node(&node) else {
+			return Ok(());
+		};
+		panel_item.backend().touch_move(uid, position);
+		Ok(())
+	}
+
+	#[doc = "Release a touch from its surface."]
+	fn touch_up(node: Arc<Node>, _calling_client: Arc<Client>, uid: u32) -> Result<()> {
+		let Some(panel_item) = panel_item_from_node(&node) else {
+			return Ok(());
+		};
+		panel_item.backend().touch_up(uid);
+		Ok(())
+	}
+
+	#[doc = "Reset all input, such as pressed keys and pointer clicks and touches. Useful for when it's newly captured into an item acceptor to make sure no input gets stuck."]
+	fn reset_input(node: Arc<Node>, _calling_client: Arc<Client>) -> Result<()> {
+		let Some(panel_item) = panel_item_from_node(&node) else {
+			return Ok(());
+		};
+		panel_item.backend().reset_input();
+		Ok(())
+	}
+}
+
+pub struct PanelItemUi;
+impl AspectIdentifier for PanelItemUi {
+	impl_aspect_for_panel_item_ui_aspect_id! {}
+}
+impl Aspect for PanelItemUi {
+	impl_aspect_for_panel_item_ui_aspect! {}
+}
+impl PanelItemUiAspect for PanelItemUi {}
+
+pub struct PanelItemAcceptor;
+impl AspectIdentifier for PanelItemAcceptor {
+	impl_aspect_for_panel_item_acceptor_aspect_id! {}
+}
+impl Aspect for PanelItemAcceptor {
+	impl_aspect_for_panel_item_acceptor_aspect! {}
+}
+impl PanelItemAcceptorAspect for PanelItemAcceptor {
+	fn capture_item(node: Arc<Node>, _calling_client: Arc<Client>, item: Arc<Node>) -> Result<()> {
+		super::acceptor_capture_item_flex(node, item)
+	}
+}
+
+impl<B: Backend> PanelItemTrait for PanelItem<B> {
+	fn backend(&self) -> &dyn Backend {
+		self.backend.as_ref()
+	}
+	fn send_ui_item_created(&self, node: &Node, item: &Arc<Node>) {
+		let Ok(init_data) = self.backend.start_data() else {
+			return;
+		};
+		let _ = panel_item_ui_client::create_item(node, item, init_data);
+	}
+	fn send_acceptor_item_created(&self, node: &Node, item: &Arc<Node>) {
+		let Ok(init_data) = self.backend.start_data() else {
+			return;
+		};
+		let _ = panel_item_acceptor_client::capture_item(node, item, init_data);
+	}
+}
+
+impl InterfaceAspect for Interface {
+	#[doc = "Register this client to manage the items of a certain type and create default 3D UI for them."]
+	fn register_panel_item_ui(node: Arc<Node>, calling_client: Arc<Client>) -> Result<()> {
+		node.add_aspect(CameraItemAcceptor);
+		register_item_ui_flex(calling_client, &ITEM_TYPE_INFO_PANEL)
+	}
+
+	#[doc = "Create an item acceptor to allow temporary ownership of a given type of item. Creates a node at `/item/<item_type>/acceptor/<name>`."]
+	fn create_panel_item_acceptor(
+		node: Arc<Node>,
+		calling_client: Arc<Client>,
+		id: u64,
+		parent: Arc<Node>,
+		transform: Transform,
+		field: Arc<Node>,
+	) -> Result<()> {
+		node.add_aspect(PanelItemAcceptor);
+		create_item_acceptor_flex(
+			calling_client,
+			id,
+			parent,
+			transform,
+			&ITEM_TYPE_INFO_PANEL,
+			field,
+		)
+	}
+
+	async fn register_keymap(
+		_node: Arc<Node>,
+		_calling_client: Arc<Client>,
+		keymap: String,
+	) -> Result<u64> {
+		let mut keymaps = KEYMAPS.lock();
+		if let Some(found_keymap_id) = keymaps
+			.iter()
+			.filter(|(_k, v)| *v == &keymap)
+			.map(|(k, _v)| k)
+			.last()
+		{
+			return Ok(found_keymap_id.data().as_ffi());
+		}
+
+		let key = keymaps.insert(keymap);
+		Ok(key.data().as_ffi())
+	}
+
+	async fn get_keymap(
+		_node: Arc<Node>,
+		_calling_client: Arc<Client>,
+		keymap_id: u64,
+	) -> Result<String> {
+		let keymaps = KEYMAPS.lock();
+		let Some(keymap) = keymaps.get(KeyData::from_ffi(keymap_id).into()) else {
+			bail!("Could not find keymap. Try registering it");
+		};
+
+		Ok(keymap.clone())
+	}
+}

src/nodes/mod.rs

@@ -1,247 +1,350 @@
 pub mod alias;
-pub mod data;
+pub mod audio;
 pub mod drawable;
 pub mod fields;
-pub mod hmd;
 pub mod input;
 pub mod items;
 pub mod root;
 pub mod spatial;
-pub mod startup;
 
-use anyhow::{anyhow, Result};
-use nanoid::nanoid;
-use once_cell::sync::OnceCell;
-use parking_lot::Mutex;
+use self::alias::Alias;
+use crate::core::client::Client;
+use crate::core::error::{Result, ServerError};
+use crate::core::registry::Registry;
+use crate::core::scenegraph::MethodResponseSender;
+use dashmap::DashMap;
+use serde::{Serialize, de::DeserializeOwned};
+use spatial::Spatial;
+use stardust_xr::messenger::MessageSenderHandle;
 use stardust_xr::scenegraph::ScenegraphError;
+use stardust_xr::schemas::flex::{deserialize, serialize};
+use std::any::{Any, TypeId};
+use std::fmt::Debug;
+use std::os::fd::OwnedFd;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::{Arc, Weak};
 use std::vec::Vec;
 
-use core::hash::BuildHasherDefault;
-use dashmap::DashMap;
-use rustc_hash::FxHasher;
-
-use crate::core::client::Client;
-use crate::core::registry::Registry;
-
-use self::alias::Alias;
-use self::data::{PulseReceiver, PulseSender};
-
-use self::drawable::model::Model;
-use self::drawable::text::Text;
-use self::fields::Field;
-use self::input::{InputHandler, InputMethod};
-use self::items::{Item, ItemAcceptor, ItemUI};
-use self::spatial::Spatial;
-use self::startup::StartupSettings;
-
-pub type Signal = fn(&Node, Arc<Client>, &[u8]) -> Result<()>;
-pub type Method = fn(&Node, Arc<Client>, &[u8]) -> Result<Vec<u8>>;
-
-pub struct Node {
-	pub(super) uid: String,
-	path: String,
-	// trailing_slash_pos: usize,
-	local_signals: DashMap<String, Signal, BuildHasherDefault<FxHasher>>,
-	local_methods: DashMap<String, Method, BuildHasherDefault<FxHasher>>,
-	destroyable: AtomicBool,
-
-	pub alias: OnceCell<Arc<Alias>>,
-	aliases: Registry<Alias>,
-
-	pub spatial: OnceCell<Arc<Spatial>>,
-	pub field: OnceCell<Arc<Field>>,
-
-	// Data
-	pub pulse_sender: OnceCell<Arc<PulseSender>>,
-	pub pulse_receiver: OnceCell<Arc<PulseReceiver>>,
-
-	// Drawable
-	pub model: OnceCell<Arc<Model>>,
-	pub text: OnceCell<Arc<Text>>,
-
-	// Input
-	pub input_method: OnceCell<Arc<InputMethod>>,
-	pub input_handler: OnceCell<Arc<InputHandler>>,
-
-	// Item
-	pub item: OnceCell<Arc<Item>>,
-	pub item_acceptor: OnceCell<Arc<ItemAcceptor>>,
-	pub item_ui: OnceCell<Arc<ItemUI>>,
-
-	// Startup
-	pub startup_settings: OnceCell<Mutex<StartupSettings>>,
-
-	pub(crate) client: Weak<Client>,
+#[derive(Default)]
+pub struct Message {
+	pub data: Vec<u8>,
+	pub fds: Vec<OwnedFd>,
+}
+impl From<Vec<u8>> for Message {
+	fn from(data: Vec<u8>) -> Self {
+		Message {
+			data,
+			fds: Vec::new(),
+		}
+	}
+}
+impl AsRef<[u8]> for Message {
+	fn as_ref(&self) -> &[u8] {
+		&self.data
+	}
 }
 
-impl Node {
-	pub fn get_client(&self) -> Option<Arc<Client>> {
-		self.client.upgrade()
-	}
-	// pub fn get_name(&self) -> &str {
-	// 	&self.path[self.trailing_slash_pos + 1..]
-	// }
-	pub fn get_path(&self) -> &str {
-		self.path.as_str()
-	}
-	pub fn is_destroyable(&self) -> bool {
-		self.destroyable.load(Ordering::Relaxed)
+stardust_xr_server_codegen::codegen_node_protocol!();
+
+pub struct Owned;
+impl AspectIdentifier for Owned {
+	impl_aspect_for_owned_aspect_id! {}
+}
+impl Aspect for Owned {
+	impl_aspect_for_owned_aspect! {}
+}
+impl OwnedAspect for Owned {
+	fn set_enabled(node: Arc<Node>, _calling_client: Arc<Client>, enabled: bool) -> Result<()> {
+		node.set_enabled(enabled);
+		Ok(())
 	}
 
-	pub fn create(client: &Arc<Client>, parent: &str, name: &str, destroyable: bool) -> Self {
-		let mut path = parent.to_string();
-		path.push('/');
-		path.push_str(name);
-		let node = Node {
-			uid: nanoid!(),
-			client: Arc::downgrade(client),
-			path,
-			// trailing_slash_pos: parent.len(),
-			local_signals: Default::default(),
-			local_methods: Default::default(),
-			destroyable: AtomicBool::from(destroyable),
-
-			alias: OnceCell::new(),
-			aliases: Registry::new(),
-
-			spatial: OnceCell::new(),
-			field: OnceCell::new(),
-			pulse_sender: OnceCell::new(),
-			pulse_receiver: OnceCell::new(),
-			model: OnceCell::new(),
-			text: OnceCell::new(),
-			input_method: OnceCell::new(),
-			input_handler: OnceCell::new(),
-			item: OnceCell::new(),
-			item_acceptor: OnceCell::new(),
-			item_ui: OnceCell::new(),
-			startup_settings: OnceCell::new(),
-		};
-		node.add_local_signal("destroy", Node::destroy_flex);
-		node
-	}
-	pub fn add_to_scenegraph(self) -> Arc<Node> {
-		self.get_client().unwrap().scenegraph.add_node(self)
-	}
-	pub fn destroy(&self) {
-		let _ = self
-			.get_client()
-			.map(|c| c.scenegraph.remove_node(self.get_path()));
-	}
-
-	pub fn destroy_flex(node: &Node, _calling_client: Arc<Client>, _data: &[u8]) -> Result<()> {
-		if node.is_destroyable() {
+	fn destroy(node: Arc<Node>, _calling_client: Arc<Client>) -> Result<()> {
+		if node.destroyable {
 			node.destroy();
 		}
 		Ok(())
 	}
+}
 
-	pub fn add_local_signal(&self, name: &str, signal: Signal) {
-		self.local_signals.insert(name.to_string(), signal);
+pub struct OwnedNode(pub Arc<Node>);
+impl Drop for OwnedNode {
+	fn drop(&mut self) {
+		self.0.destroy();
 	}
-	pub fn add_local_method(&self, name: &str, method: Method) {
-		self.local_methods.insert(name.to_string(), method);
+}
+
+pub struct Node {
+	enabled: AtomicBool,
+	id: u64,
+	client: Weak<Client>,
+	message_sender_handle: Option<MessageSenderHandle>,
+
+	aliases: Registry<Alias>,
+	aspects: Aspects,
+	destroyable: bool,
+}
+impl Node {
+	pub fn get_client(&self) -> Option<Arc<Client>> {
+		self.client.upgrade()
+	}
+	pub fn get_id(&self) -> u64 {
+		self.id
 	}
 
-	pub fn get_aspect<F, T>(
-		&self,
-		node_name: &'static str,
-		aspect_type: &'static str,
-		aspect_fn: F,
-	) -> Result<Arc<T>>
-	where
-		F: FnOnce(&Node) -> &OnceCell<Arc<T>>,
-	{
-		aspect_fn(self)
-			.get()
-			.ok_or_else(|| anyhow!("{} is not a {} node", node_name, aspect_type))
-			.cloned()
+	pub fn generate(client: &Arc<Client>, destroyable: bool) -> Self {
+		Self::from_id(client, client.generate_id(), destroyable)
+	}
+	pub fn from_id(client: &Arc<Client>, id: u64, destroyable: bool) -> Self {
+		let node = Node {
+			enabled: AtomicBool::new(true),
+			client: Arc::downgrade(client),
+			message_sender_handle: client.message_sender_handle.clone(),
+			id,
+			aliases: Default::default(),
+			aspects: Default::default(),
+			destroyable,
+		};
+		node.aspects.add(Owned);
+		node
+	}
+	pub fn add_to_scenegraph(self) -> Result<Arc<Node>> {
+		Ok(self
+			.get_client()
+			.ok_or(ServerError::NoClient)?
+			.scenegraph
+			.add_node(self))
+	}
+	pub fn add_to_scenegraph_owned(self) -> Result<OwnedNode> {
+		Ok(OwnedNode(
+			self.get_client()
+				.ok_or(ServerError::NoClient)?
+				.scenegraph
+				.add_node(self),
+		))
+	}
+	pub fn enabled(&self) -> bool {
+		self.enabled.load(Ordering::Relaxed)
+			&& if let Ok(spatial) = self.get_aspect::<Spatial>() {
+				spatial
+					.global_transform()
+					.to_scale_rotation_translation()
+					.0
+					.length_squared()
+					> 0.0
+			} else {
+				true
+			}
+	}
+	pub fn set_enabled(&self, enabled: bool) {
+		self.enabled.store(enabled, Ordering::Relaxed)
+	}
+	pub fn destroy(&self) {
+		if let Some(client) = self.get_client() {
+			client.scenegraph.remove_node(self.get_id());
+		}
+	}
+
+	// very much up for debate if we should allow this, as you can match objects using this
+	// pub fn get_client_pid_flex(
+	// 	node: Arc<Node>,
+	// 	_calling_client: Arc<Client>,
+	// 	_message: Message,
+	// ) -> Result<Message> {
+	// 	let client = node
+	// 		.client
+	// 		.upgrade()
+	// 		.ok_or_else(|| eyre!("Could not get client for node?"))?;
+	// 	let pid = client.pid.ok_or_else(|| eyre!("Client PID is unknown"))?;
+	// 	Ok(serialize(pid)?.into())
+	// }
+
+	pub fn add_aspect<A: AspectIdentifier>(&self, aspect: A) -> Arc<A> {
+		self.aspects.add(aspect)
+	}
+	pub fn add_aspect_raw<A: AspectIdentifier>(&self, aspect: Arc<A>) {
+		self.aspects.add_raw(aspect)
+	}
+	pub fn get_aspect<A: AspectIdentifier>(&self) -> Result<Arc<A>> {
+		self.aspects.get()
 	}
 
 	pub fn send_local_signal(
-		&self,
+		self: Arc<Self>,
 		calling_client: Arc<Client>,
-		method: &str,
-		data: &[u8],
+		aspect_id: u64,
+		method: u64,
+		message: Message,
 	) -> Result<(), ScenegraphError> {
-		if let Some(alias) = self.alias.get() {
-			if !alias.info.local_signals.iter().any(|e| e == &method) {
-				return Err(ScenegraphError::SignalNotFound);
+		if let Ok(alias) = self.get_aspect::<Alias>() {
+			if !alias.info.server_signals.contains(&method) {
+				return Err(ScenegraphError::MemberNotFound);
 			}
 			alias
 				.original
 				.upgrade()
 				.ok_or(ScenegraphError::BrokenAlias)?
-				.send_local_signal(calling_client, method, data)
+				.send_local_signal(calling_client, aspect_id, method, message)
 		} else {
-			let signal = self
-				.local_signals
-				.get(method)
-				.ok_or(ScenegraphError::SignalNotFound)?;
-			signal(self, calling_client, data)
-				.map_err(|error| ScenegraphError::SignalError { error })
+			let aspect = self
+				.aspects
+				.0
+				.get(&aspect_id)
+				.ok_or(ScenegraphError::AspectNotFound)?
+				.clone();
+			aspect
+				.run_signal(calling_client, self.clone(), method, message)
+				.map_err(|error| ScenegraphError::MemberError {
+					error: error.to_string(),
+				})
 		}
 	}
 	pub fn execute_local_method(
-		&self,
+		self: Arc<Self>,
 		calling_client: Arc<Client>,
-		method: &str,
-		data: &[u8],
-	) -> Result<Vec<u8>, ScenegraphError> {
-		if let Some(alias) = self.alias.get() {
-			if !alias.info.local_methods.iter().any(|e| e == &method) {
-				return Err(ScenegraphError::MethodNotFound);
+		aspect_id: u64,
+		method: u64,
+		message: Message,
+		response: MethodResponseSender,
+	) {
+		if let Ok(alias) = self.get_aspect::<Alias>() {
+			if !alias.info.server_methods.contains(&method) {
+				response.send(Err(ScenegraphError::MemberNotFound));
+				return;
 			}
-			alias
-				.original
-				.upgrade()
-				.ok_or(ScenegraphError::BrokenAlias)?
-				.execute_local_method(calling_client, method, data)
+			let Some(alias) = alias.original.upgrade() else {
+				response.send(Err(ScenegraphError::BrokenAlias));
+				return;
+			};
+			alias.execute_local_method(
+				calling_client,
+				aspect_id,
+				method,
+				Message {
+					data: message.data.clone(),
+					fds: Vec::new(),
+				},
+				response,
+			)
 		} else {
-			let method = self
-				.local_methods
-				.get(method)
-				.ok_or(ScenegraphError::MethodNotFound)?;
-			method(self, calling_client, data)
-				.map_err(|error| ScenegraphError::MethodError { error })
+			let Some(aspect) = self.aspects.0.get(&aspect_id).map(|v| v.clone()) else {
+				response.send(Err(ScenegraphError::AspectNotFound));
+				return;
+			};
+			aspect.run_method(calling_client, self.clone(), method, message, response);
 		}
 	}
-	pub fn send_remote_signal(&self, method: &str, data: &[u8]) -> Result<()> {
+	pub fn send_remote_signal(
+		&self,
+		aspect_id: u64,
+		method: u64,
+		message: impl Into<Message>,
+	) -> Result<()> {
+		let message = message.into();
 		self.aliases
 			.get_valid_contents()
 			.iter()
-			.filter(|alias| alias.info.remote_signals.iter().any(|e| e == &method))
-			.for_each(|alias| {
-				let _ = alias
-					.node
-					.upgrade()
-					.unwrap()
-					.send_remote_signal(method, data);
+			.filter(|alias| alias.info.client_signals.iter().any(|e| e == &method))
+			.filter_map(|alias| alias.node.upgrade())
+			.for_each(|node| {
+				// Beware! file descriptors will not be sent to aliases!!!
+				let _ = node.send_remote_signal(
+					aspect_id,
+					method,
+					Message {
+						data: message.data.clone(),
+						fds: Vec::new(),
+					},
+				);
 			});
-		let path = self.path.clone();
-		let method = method.to_string();
-		let data = data.to_vec();
-		if let Some(client) = self.get_client() {
-			if let Some(messenger) = client.messenger.as_ref() {
-				messenger.send_remote_signal(path.as_str(), method.as_str(), data.as_slice());
-			}
+		if let Some(handle) = self.message_sender_handle.as_ref() {
+			handle.signal(self.id, aspect_id, method, &message.data, message.fds)?;
 		}
 		Ok(())
 	}
-	pub async fn execute_remote_method(&self, method: &str, data: Vec<u8>) -> Result<Vec<u8>> {
-		if let Some(client) = self.get_client() {
-			match client.messenger.as_ref() {
-				None => Err(anyhow!("Messenger does not exist for this node's client")),
-				Some(messenger) => {
-					messenger
-						.execute_remote_method(self.path.as_str(), method, &data)
-						.await
-				}
-			}
-		} else {
-			Err(anyhow!("Client does not exist somehow?"))
-		}
+	pub async fn execute_remote_method_typed<S: Serialize, D: DeserializeOwned>(
+		&self,
+		aspect_id: u64,
+		method: u64,
+		input: S,
+		fds: Vec<OwnedFd>,
+	) -> Result<(D, Vec<OwnedFd>)> {
+		let message_sender_handle = self
+			.message_sender_handle
+			.as_ref()
+			.ok_or(ServerError::NoMessenger)?;
+
+		let serialized = serialize(input)?;
+		let result = message_sender_handle
+			.method(self.id, aspect_id, method, &serialized, fds)
+			.await?
+			.map_err(ServerError::RemoteMethodError)?;
+
+		let (message, fds) = result.into_components();
+		let deserialized: D = deserialize(&message)?;
+		Ok((deserialized, fds))
+	}
+}
+impl Debug for Node {
+	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+		f.debug_struct("Node")
+			.field("id", &self.id)
+			.field("destroyable", &self.destroyable)
+			.finish()
+	}
+}
+impl Drop for Node {
+	fn drop(&mut self) {
+		// Debug breakpoint
+	}
+}
+
+pub trait AspectIdentifier: Aspect {
+	const ID: u64;
+}
+pub trait Aspect: Any + Send + Sync + 'static {
+	fn as_any(self: Arc<Self>) -> Arc<dyn Any + Send + Sync + 'static>;
+	fn run_signal(
+		&self,
+		calling_client: Arc<Client>,
+		node: Arc<Node>,
+		signal: u64,
+		message: Message,
+	) -> Result<(), stardust_xr::scenegraph::ScenegraphError>;
+	fn run_method(
+		&self,
+		calling_client: Arc<Client>,
+		node: Arc<Node>,
+		method: u64,
+		message: Message,
+		response: MethodResponseSender,
+	);
+}
+
+#[derive(Default)]
+struct Aspects(DashMap<u64, Arc<dyn Aspect>>);
+impl Aspects {
+	fn add<A: AspectIdentifier>(&self, t: A) -> Arc<A> {
+		let aspect = Arc::new(t);
+		self.add_raw(aspect.clone());
+		aspect
+	}
+	fn add_raw<A: AspectIdentifier>(&self, aspect: Arc<A>) {
+		self.0.insert(A::ID, aspect);
+	}
+	fn get<A: Aspect + AspectIdentifier>(&self) -> Result<Arc<A>> {
+		self.0
+			.get(&A::ID)
+			// .cloned doesn't work for some reason
+			.map(|v| v.clone())
+			.map(|a| a.as_any())
+			.and_then(|a| Arc::downcast(a).ok())
+			.ok_or(ServerError::NoAspect(TypeId::of::<A>()))
+	}
+}
+impl Drop for Aspects {
+	fn drop(&mut self) {
+		// why would this be needed? do drop impls not run otherwise?
+		self.0.clear()
 	}
 }

src/nodes/root.rs

@@ -1,81 +1,105 @@
 use super::spatial::Spatial;
-use super::startup::DESKTOP_STARTUP_IDS;
-use super::Node;
-use crate::core::client::Client;
-use crate::core::registry::Registry;
-use anyhow::{anyhow, Result};
+use super::{Aspect, AspectIdentifier, Node};
+use crate::bail;
+use crate::core::client::{CLIENTS, Client};
+use crate::core::client_state::ClientStateParsed;
+use crate::core::error::Result;
+use crate::nodes::spatial::SPATIAL_REF_ASPECT_ALIAS_INFO;
+use crate::session::connection_env;
 use glam::Mat4;
-use stardust_xr::schemas::flex::{deserialize, serialize};
-
-use std::sync::atomic::{AtomicBool, Ordering};
+use std::path::PathBuf;
 use std::sync::Arc;
+use std::time::Instant;
+use tracing::info;
 
-static ROOT_REGISTRY: Registry<Root> = Registry::new();
+stardust_xr_server_codegen::codegen_root_protocol!();
 
 pub struct Root {
 	node: Arc<Node>,
-	logic_step: AtomicBool,
+	connect_instant: Instant,
 }
 impl Root {
-	pub fn create(client: &Arc<Client>) -> Arc<Self> {
-		let node = Node::create(client, "", "", false);
-		node.add_local_signal("applyDesktopStartupID", Root::apply_desktop_startup_id);
-		node.add_local_signal("subscribeLogicStep", Root::subscribe_logic_step);
-		node.add_local_signal("setBasePrefixes", Root::set_base_prefixes);
-		let node = node.add_to_scenegraph();
-		let _ = Spatial::add_to(&node, None, Mat4::IDENTITY);
-
-		ROOT_REGISTRY.add(Root {
-			node,
-			logic_step: AtomicBool::from(false),
-		})
+	pub fn create(client: &Arc<Client>, transform: Mat4) -> Result<Arc<Self>> {
+		let node = Node::from_id(client, 0, false);
+		let node = node.add_to_scenegraph()?;
+		let _ = Spatial::add_to(&node, None, transform, false);
+		let root_aspect = node.add_aspect(Root {
+			node: node.clone(),
+			connect_instant: Instant::now(),
+		});
+		Ok(root_aspect)
 	}
 
-	fn apply_desktop_startup_id(
-		node: &Node,
-		_calling_client: Arc<Client>,
-		data: &[u8],
-	) -> Result<()> {
-		let startup_settings = DESKTOP_STARTUP_IDS
-			.lock()
-			.remove(flexbuffers::Reader::get_root(data)?.get_str()?)
-			.ok_or_else(|| anyhow!("Desktop startup ID not found in the list!"))?;
-		node.spatial
-			.get()
-			.unwrap()
-			.set_local_transform(startup_settings.transform);
-
-		Ok(())
-	}
-
-	fn subscribe_logic_step(_node: &Node, calling_client: Arc<Client>, _data: &[u8]) -> Result<()> {
-		calling_client
-			.root
-			.get()
-			.unwrap()
-			.logic_step
-			.store(true, Ordering::Relaxed);
-		Ok(())
-	}
-
-	pub fn logic_step(delta: f64) {
-		if let Ok(data) = serialize((delta, 0.0)) {
-			for root in ROOT_REGISTRY.get_valid_contents() {
-				if root.logic_step.load(Ordering::Relaxed) {
-					let _ = root.node.send_remote_signal("logicStep", &data);
-				}
-			}
+	pub fn send_frame_events(delta: f64) {
+		for client in CLIENTS.get_vec() {
+			let Some(root) = client.root.get() else {
+				continue;
+			};
+			let _ = root_client::frame(
+				&root.node,
+				&FrameInfo {
+					delta: delta as f32,
+					elapsed: root.connect_instant.elapsed().as_secs_f32(),
+				},
+			);
 		}
 	}
 
-	fn set_base_prefixes(_node: &Node, calling_client: Arc<Client>, data: &[u8]) -> Result<()> {
-		*calling_client.base_resource_prefixes.lock() = deserialize(data)?;
+	pub fn set_transform(&self, transform: Mat4) {
+		let spatial = self.node.get_aspect::<Spatial>().unwrap();
+		// spatial.set_spatial_parent(None).unwrap();
+		spatial.set_local_transform(transform);
+	}
+	pub async fn save_state(&self) -> Result<ClientState> {
+		Ok(root_client::save_state(&self.node).await?.0)
+	}
+}
+impl AspectIdentifier for Root {
+	impl_aspect_for_root_aspect_id! {}
+}
+impl Aspect for Root {
+	impl_aspect_for_root_aspect! {}
+}
+impl RootAspect for Root {
+	async fn get_state(_node: Arc<Node>, calling_client: Arc<Client>) -> Result<ClientState> {
+		let Some(state) = calling_client.state.get() else {
+			bail!("Couldn't get state");
+		};
+		Ok(state.clone())
+	}
+
+	#[doc = "Get a hashmap of all the environment variables to connect a given app to the stardust server"]
+	async fn get_connection_environment(
+		_node: Arc<Node>,
+		_calling_client: Arc<Client>,
+	) -> Result<stardust_xr::values::Map<String, String>> {
+		Ok(connection_env())
+	}
+
+	#[doc = "Generate a client state token and return it back.\n\n When launching a new client, set the environment variable `STARDUST_STARTUP_TOKEN` to the returned string.\n Make sure the environment variable shows in `/proc/{pid}/environ` as that's the only reliable way to pass the value to the server (suggestions welcome).\n"]
+	async fn generate_state_token(
+		_node: Arc<Node>,
+		calling_client: Arc<Client>,
+		state: ClientState,
+	) -> Result<String> {
+		Ok(ClientStateParsed::from_deserialized(&calling_client, state).token())
+	}
+
+	#[doc = "Set initial list of folders to look for namespaced resources in"]
+	fn set_base_prefixes(
+		_node: Arc<Node>,
+		calling_client: Arc<Client>,
+		prefixes: Vec<String>,
+	) -> Result<()> {
+		info!(?calling_client, ?prefixes, "Set base prefixes");
+		*calling_client.base_resource_prefixes.lock() =
+			prefixes.into_iter().map(PathBuf::from).collect();
+		Ok(())
+	}
+
+	#[doc = "Cleanly disconnect from the server"]
+	fn disconnect(_node: Arc<Node>, calling_client: Arc<Client>) -> Result<()> {
+		calling_client.disconnect(Ok(()));
 		Ok(())
 	}
 }
-
-impl Drop for Root {
-	fn drop(&mut self) {
-		ROOT_REGISTRY.remove(self);
-	}
-}

src/nodes/spatial.rs

@@ -1,291 +0,0 @@
-use super::Node;
-use crate::core::client::Client;
-use anyhow::{anyhow, ensure, Result};
-use glam::{Mat4, Quat};
-use mint::Vector3;
-use parking_lot::Mutex;
-
-use serde::Deserialize;
-use stardust_xr::schemas::flex::{deserialize, serialize};
-use stardust_xr::values::Transform;
-use std::ptr;
-use std::sync::{Arc, Weak};
-
-pub struct Spatial {
-	pub(super) node: Weak<Node>,
-	parent: Mutex<Option<Arc<Spatial>>>,
-	pub(super) transform: Mutex<Mat4>,
-}
-
-impl Spatial {
-	pub fn new(node: Weak<Node>, parent: Option<Arc<Spatial>>, transform: Mat4) -> Arc<Self> {
-		Arc::new(Spatial {
-			node,
-			parent: Mutex::new(parent),
-			transform: Mutex::new(transform),
-		})
-	}
-	pub fn add_to(
-		node: &Arc<Node>,
-		parent: Option<Arc<Spatial>>,
-		transform: Mat4,
-	) -> Result<Arc<Spatial>> {
-		ensure!(
-			node.spatial.get().is_none(),
-			"Internal: Node already has a Spatial aspect!"
-		);
-		let spatial = Spatial {
-			node: Arc::downgrade(node),
-			parent: Mutex::new(parent),
-			transform: Mutex::new(transform),
-		};
-		node.add_local_method("getTransform", Spatial::get_transform_flex);
-		node.add_local_signal("setTransform", Spatial::set_transform_flex);
-		node.add_local_signal("setSpatialParent", Spatial::set_spatial_parent_flex);
-		node.add_local_signal(
-			"setSpatialParentInPlace",
-			Spatial::set_spatial_parent_in_place_flex,
-		);
-		let spatial_arc = Arc::new(spatial);
-		let _ = node.spatial.set(spatial_arc.clone());
-		Ok(spatial_arc)
-	}
-
-	pub fn space_to_space_matrix(from: Option<&Spatial>, to: Option<&Spatial>) -> Mat4 {
-		let space_to_world_matrix = from.map_or(Mat4::IDENTITY, |from| from.global_transform());
-		let world_to_space_matrix = to.map_or(Mat4::IDENTITY, |to| to.global_transform().inverse());
-		world_to_space_matrix * space_to_world_matrix
-	}
-
-	pub fn local_transform(&self) -> Mat4 {
-		*self.transform.lock()
-	}
-	pub fn global_transform(&self) -> Mat4 {
-		match self.parent.lock().clone() {
-			Some(value) => value.global_transform() * *self.transform.lock(),
-			None => *self.transform.lock(),
-		}
-	}
-	pub fn set_local_transform(&self, transform: Mat4) {
-		*self.transform.lock() = transform;
-	}
-	pub fn set_local_transform_components(
-		&self,
-		reference_space: Option<&Spatial>,
-		transform: Transform,
-	) {
-		let reference_to_parent_transform = reference_space
-			.map(|reference_space| {
-				Spatial::space_to_space_matrix(Some(reference_space), self.parent.lock().as_deref())
-			})
-			.unwrap_or(Mat4::IDENTITY);
-		let mut local_transform_in_reference_space =
-			reference_to_parent_transform.inverse() * self.local_transform();
-		let (mut reference_space_scl, mut reference_space_rot, mut reference_space_pos) =
-			local_transform_in_reference_space.to_scale_rotation_translation();
-
-		if let Some(pos) = transform.position {
-			reference_space_pos = pos.into()
-		}
-		if let Some(rot) = transform.rotation {
-			reference_space_rot = rot.into()
-		} else if reference_space_rot.is_nan() {
-			reference_space_rot = Quat::IDENTITY;
-		}
-		if let Some(scl) = transform.scale {
-			reference_space_scl = scl.into()
-		}
-
-		local_transform_in_reference_space = Mat4::from_scale_rotation_translation(
-			reference_space_scl,
-			reference_space_rot,
-			reference_space_pos,
-		);
-		self.set_local_transform(
-			reference_to_parent_transform * local_transform_in_reference_space,
-		);
-	}
-
-	pub fn is_ancestor_of(&self, spatial: Arc<Spatial>) -> bool {
-		let mut current_ancestor = spatial;
-		loop {
-			if Arc::as_ptr(&current_ancestor) == ptr::addr_of!(*self) {
-				return true;
-			}
-
-			let current_ancestor_parent = current_ancestor.parent.lock().clone();
-			if let Some(parent) = current_ancestor_parent {
-				current_ancestor = parent;
-			} else {
-				return false;
-			}
-		}
-	}
-
-	pub fn set_spatial_parent(&self, parent: Option<&Arc<Spatial>>) -> Result<()> {
-		let is_ancestor = parent
-			.map(|parent| self.is_ancestor_of(parent.clone()))
-			.unwrap_or(false);
-		if is_ancestor {
-			return Err(anyhow!("Setting spatial parent would cause a loop"));
-		}
-
-		*self.parent.lock() = parent.cloned();
-
-		Ok(())
-	}
-
-	pub fn set_spatial_parent_in_place(&self, parent: Option<&Arc<Spatial>>) -> Result<()> {
-		let is_ancestor = parent
-			.map(|parent| self.is_ancestor_of(parent.clone()))
-			.unwrap_or(false);
-		if is_ancestor {
-			return Err(anyhow!("Setting spatial parent would cause a loop"));
-		}
-
-		self.set_local_transform(Spatial::space_to_space_matrix(
-			Some(self),
-			parent.cloned().as_deref(),
-		));
-		*self.parent.lock() = parent.cloned();
-
-		Ok(())
-	}
-
-	pub fn get_transform_flex(
-		node: &Node,
-		calling_client: Arc<Client>,
-		data: &[u8],
-	) -> Result<Vec<u8>> {
-		let this_spatial = node
-			.spatial
-			.get()
-			.ok_or_else(|| anyhow!("Node doesn't have a spatial?"))?;
-		let relative_spatial = find_reference_space(&calling_client, deserialize(data)?)?;
-
-		let (scale, rotation, position) = Spatial::space_to_space_matrix(
-			Some(this_spatial.as_ref()),
-			Some(relative_spatial.as_ref()),
-		)
-		.to_scale_rotation_translation();
-
-		serialize((
-			mint::Vector3::from(position),
-			mint::Quaternion::from(rotation),
-			mint::Vector3::from(scale),
-		))
-		.map_err(|e| e.into())
-	}
-	pub fn set_transform_flex(node: &Node, calling_client: Arc<Client>, data: &[u8]) -> Result<()> {
-		#[derive(Deserialize)]
-		struct TransformArgs<'a> {
-			reference_space_path: Option<&'a str>,
-			transform: Transform,
-		}
-		let transform_args: TransformArgs = deserialize(data)?;
-		let reference_space_transform = transform_args
-			.reference_space_path
-			.map(|path| find_reference_space(&calling_client, path))
-			.transpose()?;
-
-		node.spatial.get().unwrap().set_local_transform_components(
-			reference_space_transform.as_deref(),
-			transform_args.transform,
-		);
-		Ok(())
-	}
-	pub fn set_spatial_parent_flex(
-		node: &Node,
-		calling_client: Arc<Client>,
-		data: &[u8],
-	) -> Result<()> {
-		let parent = find_spatial_parent(&calling_client, deserialize(data)?)?;
-		node.spatial
-			.get()
-			.unwrap()
-			.set_spatial_parent(Some(&parent))
-	}
-	pub fn set_spatial_parent_in_place_flex(
-		node: &Node,
-		calling_client: Arc<Client>,
-		data: &[u8],
-	) -> Result<()> {
-		let parent = find_spatial_parent(&calling_client, deserialize(data)?)?;
-		node.spatial
-			.get()
-			.unwrap()
-			.set_spatial_parent_in_place(Some(&parent))?;
-		Ok(())
-	}
-}
-
-pub fn parse_transform(
-	transform: Transform,
-	translation: bool,
-	rotation: bool,
-	scale: bool,
-) -> Result<Mat4> {
-	let translation = translation
-		.then_some(transform.position)
-		.flatten()
-		.unwrap_or_else(|| Vector3::from([0.0; 3]));
-	let rotation = rotation
-		.then_some(transform.rotation)
-		.flatten()
-		.unwrap_or_else(|| Quat::IDENTITY.into());
-	let scale = scale
-		.then_some(transform.scale)
-		.flatten()
-		.unwrap_or_else(|| Vector3::from([1.0; 3]));
-
-	Ok(Mat4::from_scale_rotation_translation(
-		scale.into(),
-		rotation.into(),
-		translation.into(),
-	))
-}
-
-pub fn find_spatial(
-	calling_client: &Arc<Client>,
-	node_name: &'static str,
-	node_path: &str,
-) -> anyhow::Result<Arc<Spatial>> {
-	Ok(calling_client
-		.get_node(node_name, node_path)?
-		.get_aspect(node_name, "spatial", |n| &n.spatial)?
-		.clone())
-}
-pub fn find_spatial_parent(
-	calling_client: &Arc<Client>,
-	node_path: &str,
-) -> anyhow::Result<Arc<Spatial>> {
-	find_spatial(calling_client, "Spatial parent", node_path)
-}
-pub fn find_reference_space(
-	calling_client: &Arc<Client>,
-	node_path: &str,
-) -> anyhow::Result<Arc<Spatial>> {
-	find_spatial(calling_client, "Reference space", node_path)
-}
-
-pub fn create_interface(client: &Arc<Client>) {
-	let node = Node::create(client, "", "spatial", false);
-	node.add_local_signal("createSpatial", create_spatial_flex);
-	node.add_to_scenegraph();
-}
-
-pub fn create_spatial_flex(_node: &Node, calling_client: Arc<Client>, data: &[u8]) -> Result<()> {
-	#[derive(Deserialize)]
-	struct CreateSpatialInfo<'a> {
-		name: &'a str,
-		parent_path: &'a str,
-		transform: Transform,
-	}
-	let info: CreateSpatialInfo = deserialize(data)?;
-	let node = Node::create(&calling_client, "/spatial/spatial", info.name, true);
-	let parent = find_spatial_parent(&calling_client, info.parent_path)?;
-	let transform = parse_transform(info.transform, true, true, true)?;
-	let node = node.add_to_scenegraph();
-	Spatial::add_to(&node, Some(parent), transform)?;
-	Ok(())
-}

src/nodes/spatial/mod.rs

@@ -0,0 +1,536 @@
+pub mod zone;
+
+use self::zone::Zone;
+use super::alias::Alias;
+use super::fields::{Field, FieldTrait};
+use super::{Aspect, AspectIdentifier};
+use crate::bail;
+use crate::core::client::Client;
+use crate::core::error::Result;
+use crate::core::registry::Registry;
+use crate::nodes::{Node, OWNED_ASPECT_ALIAS_INFO};
+use bevy::prelude::Transform as BevyTransform;
+use bevy::prelude::*;
+use bevy::render::primitives::Aabb;
+use color_eyre::eyre::OptionExt;
+use glam::{Mat4, Quat, Vec3, vec3a};
+use mint::Vector3;
+use parking_lot::Mutex;
+use rustc_hash::FxHashMap;
+use std::fmt::Debug;
+use std::sync::atomic::Ordering;
+use std::sync::{Arc, OnceLock, Weak};
+use std::{f32, ptr};
+
+pub struct SpatialNodePlugin;
+impl Plugin for SpatialNodePlugin {
+	fn build(&self, app: &mut App) {
+		app.add_systems(
+			PostUpdate,
+			update_spatial_nodes.before(TransformSystem::TransformPropagate),
+		);
+	}
+}
+
+fn update_spatial_nodes(
+	mut query: Query<(
+		&mut BevyTransform,
+		&SpatialNode,
+		Option<&ChildOf>,
+		&mut Visibility,
+	)>,
+	parent_query: Query<&GlobalTransform>,
+) {
+	query
+		.par_iter_mut()
+		.for_each(|(mut transform, spatial_node, child_of, mut vis)| {
+			let _span = debug_span!("updating spatial node").entered();
+			let Some(spatial) = spatial_node.0.upgrade() else {
+				return;
+			};
+			if spatial
+				.node()
+				.is_some_and(|v| !v.enabled.load(Ordering::Relaxed))
+			{
+				if !matches!(*vis, Visibility::Hidden) {
+					*vis = Visibility::Hidden;
+				}
+				return;
+			}
+			let mat4 =
+				debug_span!("getting global transform").in_scope(|| spatial.global_transform());
+			let (scale, _, _) = mat4.to_scale_rotation_translation();
+			match (*vis, scale == Vec3::ZERO) {
+				(Visibility::Inherited | Visibility::Visible, true) => {
+					*vis = Visibility::Hidden;
+				}
+				(Visibility::Hidden, false) => {
+					*vis = Visibility::Inherited;
+				}
+				_ => {}
+			}
+			match child_of {
+				Some(child_of) => {
+					let Ok(parent) = parent_query.get(child_of.0) else {
+						warn!("SpatialNode bevy Parent doesn't have global transform");
+						return;
+					};
+					*transform =
+						BevyTransform::from_matrix(parent.compute_matrix().inverse() * mat4);
+				}
+				None => {
+					*transform = BevyTransform::from_matrix(mat4);
+				}
+			}
+		});
+}
+
+#[derive(Clone, Component, Debug)]
+#[require(BevyTransform, Visibility)]
+pub struct SpatialNode(pub Weak<Spatial>);
+
+stardust_xr_server_codegen::codegen_spatial_protocol!();
+impl Transform {
+	pub fn to_mat4(&self, position: bool, rotation: bool, scale: bool) -> Mat4 {
+		let position = position
+			.then_some(self.translation)
+			.flatten()
+			.unwrap_or_else(|| Vector3::from([0.0; 3]));
+		let rotation = rotation
+			.then_some(self.rotation)
+			.flatten()
+			.unwrap_or_else(|| Quat::IDENTITY.into());
+		let scale = scale
+			.then_some(self.scale)
+			.flatten()
+			.unwrap_or_else(|| Vector3::from([1.0; 3]));
+
+		Mat4::from_scale_rotation_translation(scale.into(), rotation.into(), position.into())
+	}
+}
+impl AspectIdentifier for Zone {
+	impl_aspect_for_zone_aspect_id! {}
+}
+impl Aspect for Zone {
+	impl_aspect_for_zone_aspect! {}
+}
+
+lazy_static::lazy_static! {
+	pub static ref EXPORTED_SPATIALS: Mutex<FxHashMap<u64, Weak<Node>>> = Mutex::new(FxHashMap::default());
+}
+
+static ZONEABLE_REGISTRY: Registry<Spatial> = Registry::new();
+
+pub struct Spatial {
+	pub node: Weak<Node>,
+	parent: Mutex<Option<Arc<Spatial>>>,
+	old_parent: Mutex<Option<Arc<Spatial>>>,
+	transform: Mutex<Mat4>,
+	zone: Mutex<Weak<Zone>>,
+	children: Registry<Spatial>,
+	pub bounding_box_calc: OnceLock<fn(&Node) -> Aabb>,
+}
+
+impl Spatial {
+	pub fn new(node: Weak<Node>, parent: Option<Arc<Spatial>>, transform: Mat4) -> Arc<Self> {
+		Arc::new(Spatial {
+			node,
+			parent: Mutex::new(parent),
+			old_parent: Mutex::new(None),
+			transform: Mutex::new(transform),
+			zone: Mutex::new(Weak::new()),
+			children: Registry::new(),
+			bounding_box_calc: OnceLock::default(),
+		})
+	}
+	pub fn add_to(
+		node: &Arc<Node>,
+		parent: Option<Arc<Spatial>>,
+		transform: Mat4,
+		zoneable: bool,
+	) -> Arc<Spatial> {
+		let spatial = Spatial::new(Arc::downgrade(node), parent.clone(), transform);
+		if zoneable {
+			ZONEABLE_REGISTRY.add_raw(&spatial);
+		}
+		if let Some(parent) = parent {
+			parent.children.add_raw(&spatial);
+		}
+		node.add_aspect_raw(spatial.clone());
+		node.add_aspect(SpatialRef);
+		spatial
+	}
+
+	pub fn node(&self) -> Option<Arc<Node>> {
+		self.node.upgrade()
+	}
+
+	pub fn space_to_space_matrix(from: Option<&Spatial>, to: Option<&Spatial>) -> Mat4 {
+		let space_to_world_matrix = from.map_or(Mat4::IDENTITY, |from| from.global_transform());
+		let world_to_space_matrix = to.map_or(Mat4::IDENTITY, |to| to.global_transform().inverse());
+		world_to_space_matrix * space_to_world_matrix
+	}
+
+	// the output bounds are probably way bigger than they need to be
+	pub fn get_bounding_box(&self) -> Aabb {
+		let Some(node) = self.node() else {
+			return Aabb::default();
+		};
+		let mut bounds = self
+			.bounding_box_calc
+			.get()
+			.map(|b| (b)(&node))
+			.unwrap_or_default();
+		for child in self.children.get_valid_contents() {
+			let mat = child.local_transform();
+			let child_aabb = child.get_bounding_box();
+			bounds = Aabb::enclosing([
+				bounds.min().into(),
+				bounds.max().into(),
+				mat.transform_point3(child_aabb.min().into()),
+				mat.transform_point3(child_aabb.max().into()),
+			])
+			.unwrap();
+		}
+		bounds
+	}
+
+	pub fn local_transform(&self) -> Mat4 {
+		*self.transform.lock()
+	}
+	pub fn global_transform(&self) -> Mat4 {
+		let parent_transform = self
+			.get_parent()
+			.as_deref()
+			.map(Self::global_transform)
+			.unwrap_or_default();
+		parent_transform * self.local_transform()
+	}
+	pub fn set_local_transform(&self, transform: Mat4) {
+		*self.transform.lock() = transform;
+	}
+	pub fn set_local_transform_components(
+		&self,
+		reference_space: Option<&Spatial>,
+		transform: Transform,
+	) {
+		if reference_space == Some(self) {
+			self.set_local_transform(
+				parse_transform(transform, true, true, true) * self.local_transform(),
+			);
+			return;
+		}
+		let reference_to_parent_transform = reference_space
+			.map(|reference_space| {
+				Spatial::space_to_space_matrix(Some(reference_space), self.get_parent().as_deref())
+			})
+			.unwrap_or(Mat4::IDENTITY);
+		let mut local_transform_in_reference_space =
+			reference_to_parent_transform.inverse() * self.local_transform();
+		let (mut reference_space_scl, mut reference_space_rot, mut reference_space_pos) =
+			local_transform_in_reference_space.to_scale_rotation_translation();
+
+		if let Some(pos) = transform.translation {
+			reference_space_pos = pos.into()
+		}
+		if let Some(rot) = transform.rotation {
+			reference_space_rot = rot.into()
+		} else if reference_space_rot.is_nan() {
+			reference_space_rot = Quat::IDENTITY;
+		}
+		if let Some(scl) = transform.scale {
+			reference_space_scl = scl.into()
+		}
+
+		local_transform_in_reference_space = Mat4::from_scale_rotation_translation(
+			reference_space_scl,
+			reference_space_rot,
+			reference_space_pos,
+		);
+		self.set_local_transform(
+			reference_to_parent_transform * local_transform_in_reference_space,
+		);
+	}
+
+	pub fn is_ancestor_of(&self, spatial: Arc<Spatial>) -> bool {
+		let mut current_ancestor = spatial;
+		loop {
+			if Arc::as_ptr(&current_ancestor) == ptr::addr_of!(*self) {
+				return true;
+			}
+
+			if let Some(parent) = current_ancestor.get_parent() {
+				current_ancestor = parent;
+			} else {
+				return false;
+			}
+		}
+	}
+
+	fn get_parent(&self) -> Option<Arc<Spatial>> {
+		self.parent.lock().clone()
+	}
+	fn set_parent(self: &Arc<Self>, new_parent: &Arc<Spatial>) {
+		if let Some(parent) = self.get_parent() {
+			parent.children.remove(self);
+		}
+		new_parent.children.add_raw(self);
+
+		*self.parent.lock() = Some(new_parent.clone());
+	}
+
+	pub fn set_spatial_parent(self: &Arc<Self>, parent: &Arc<Spatial>) -> Result<()> {
+		if self.is_ancestor_of(parent.clone()) {
+			bail!("Setting spatial parent would cause a loop");
+		}
+		self.set_parent(parent);
+
+		Ok(())
+	}
+	pub fn set_spatial_parent_in_place(self: &Arc<Self>, parent: &Arc<Spatial>) -> Result<()> {
+		if self.is_ancestor_of(parent.clone()) {
+			bail!("Setting spatial parent would cause a loop");
+		}
+
+		self.set_local_transform(Spatial::space_to_space_matrix(Some(self), Some(parent)));
+		self.set_parent(parent);
+
+		Ok(())
+	}
+
+	pub(self) fn zone_distance(&self) -> f32 {
+		self.zone
+			.lock()
+			.upgrade()
+			.map(|zone| zone.field.clone())
+			.map(|field| field.distance(self, vec3a(0.0, 0.0, 0.0)))
+			.unwrap_or(f32::NEG_INFINITY)
+	}
+}
+impl AspectIdentifier for Spatial {
+	impl_aspect_for_spatial_aspect_id! {}
+}
+impl Aspect for Spatial {
+	impl_aspect_for_spatial_aspect! {}
+}
+impl SpatialAspect for Spatial {
+	fn set_local_transform(
+		node: Arc<Node>,
+		_calling_client: Arc<Client>,
+		transform: Transform,
+	) -> Result<()> {
+		let this_spatial = node.get_aspect::<Spatial>()?;
+		this_spatial.set_local_transform_components(None, transform);
+		Ok(())
+	}
+	fn set_relative_transform(
+		node: Arc<Node>,
+		_calling_client: Arc<Client>,
+		relative_to: Arc<Node>,
+		transform: Transform,
+	) -> Result<()> {
+		let this_spatial = node.get_aspect::<Spatial>()?;
+		let relative_spatial = relative_to.get_aspect::<Spatial>()?;
+
+		this_spatial.set_local_transform_components(Some(&relative_spatial), transform);
+		Ok(())
+	}
+
+	fn set_spatial_parent(
+		node: Arc<Node>,
+		_calling_client: Arc<Client>,
+		parent: Arc<Node>,
+	) -> Result<()> {
+		let this_spatial = node.get_aspect::<Spatial>()?;
+		let parent = parent.get_aspect::<Spatial>()?;
+
+		this_spatial.set_spatial_parent(&parent)?;
+		Ok(())
+	}
+
+	fn set_spatial_parent_in_place(
+		node: Arc<Node>,
+		_calling_client: Arc<Client>,
+		parent: Arc<Node>,
+	) -> Result<()> {
+		let this_spatial = node.get_aspect::<Spatial>()?;
+		let parent = parent.get_aspect::<Spatial>()?;
+
+		this_spatial.set_spatial_parent_in_place(&parent)?;
+		Ok(())
+	}
+
+	fn set_zoneable(node: Arc<Node>, _calling_client: Arc<Client>, zoneable: bool) -> Result<()> {
+		let spatial = node.get_aspect::<Spatial>()?;
+		if zoneable {
+			ZONEABLE_REGISTRY.add_raw(&spatial);
+		} else {
+			ZONEABLE_REGISTRY.remove(&spatial);
+			zone::release(&spatial);
+		}
+		Ok(())
+	}
+
+	// legit gotta find a way to remove old ones, this just keeps the node alive
+	async fn export_spatial(node: Arc<Node>, _calling_client: Arc<Client>) -> Result<u64> {
+		let id = rand::random();
+		EXPORTED_SPATIALS.lock().insert(id, Arc::downgrade(&node));
+		Ok(id)
+	}
+}
+impl PartialEq for Spatial {
+	fn eq(&self, other: &Self) -> bool {
+		self.node.as_ptr() == other.node.as_ptr()
+	}
+}
+impl Debug for Spatial {
+	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+		f.debug_struct("Spatial")
+			.field("parent", &self.parent)
+			.field("old_parent", &self.old_parent)
+			.field("transform", &self.transform)
+			.finish()
+	}
+}
+impl Drop for Spatial {
+	fn drop(&mut self) {
+		zone::release(self);
+		ZONEABLE_REGISTRY.remove(self);
+	}
+}
+
+pub struct SpatialRef;
+impl AspectIdentifier for SpatialRef {
+	impl_aspect_for_spatial_ref_aspect_id! {}
+}
+impl Aspect for SpatialRef {
+	impl_aspect_for_spatial_ref_aspect! {}
+}
+impl SpatialRefAspect for SpatialRef {
+	async fn get_local_bounding_box(
+		node: Arc<Node>,
+		_calling_client: Arc<Client>,
+	) -> Result<BoundingBox> {
+		let this_spatial = node.get_aspect::<Spatial>()?;
+		let bounds = this_spatial.get_bounding_box();
+
+		Ok(BoundingBox {
+			center: Vec3::from(bounds.center).into(),
+			size: Vec3::from(bounds.half_extents * 2.0).into(),
+		})
+	}
+
+	async fn get_relative_bounding_box(
+		node: Arc<Node>,
+		_calling_client: Arc<Client>,
+		relative_to: Arc<Node>,
+	) -> Result<BoundingBox> {
+		let this_spatial = node.get_aspect::<Spatial>()?;
+		let relative_spatial = relative_to.get_aspect::<Spatial>()?;
+		let mat = Spatial::space_to_space_matrix(Some(&this_spatial), Some(&relative_spatial));
+		let bb = this_spatial.get_bounding_box();
+		let bounds = Aabb::enclosing([
+			mat.transform_point3(bb.min().into()),
+			mat.transform_point3(bb.max().into()),
+		])
+		.unwrap();
+
+		Ok(BoundingBox {
+			center: Vec3::from(bounds.center).into(),
+			size: Vec3::from(bounds.half_extents * 2.0).into(),
+		})
+	}
+
+	async fn get_transform(
+		node: Arc<Node>,
+		_calling_client: Arc<Client>,
+		relative_to: Arc<Node>,
+	) -> Result<Transform> {
+		let this_spatial = node.get_aspect::<Spatial>()?;
+		let relative_spatial = relative_to.get_aspect::<Spatial>()?;
+
+		let (scale, rotation, position) = Spatial::space_to_space_matrix(
+			Some(this_spatial.as_ref()),
+			Some(relative_spatial.as_ref()),
+		)
+		.to_scale_rotation_translation();
+
+		Ok(Transform {
+			translation: Some(position.into()),
+			rotation: Some(rotation.into()),
+			scale: Some(scale.into()),
+		})
+	}
+}
+
+pub fn parse_transform(transform: Transform, position: bool, rotation: bool, scale: bool) -> Mat4 {
+	let position = position
+		.then_some(transform.translation)
+		.flatten()
+		.unwrap_or_else(|| Vector3::from([0.0; 3]));
+	let rotation = rotation
+		.then_some(transform.rotation)
+		.flatten()
+		.unwrap_or_else(|| Quat::IDENTITY.into());
+	let scale = scale
+		.then_some(transform.scale)
+		.flatten()
+		.unwrap_or_else(|| Vector3::from([1.0; 3]));
+
+	Mat4::from_scale_rotation_translation(scale.into(), rotation.into(), position.into())
+}
+
+impl InterfaceAspect for Interface {
+	fn create_spatial(
+		_node: Arc<Node>,
+		calling_client: Arc<Client>,
+		id: u64,
+		parent: Arc<Node>,
+		transform: Transform,
+		zoneable: bool,
+	) -> Result<()> {
+		let parent = parent.get_aspect::<Spatial>()?;
+		let transform = parse_transform(transform, true, true, true);
+		let node = Node::from_id(&calling_client, id, true).add_to_scenegraph()?;
+		Spatial::add_to(&node, Some(parent.clone()), transform, zoneable);
+		Ok(())
+	}
+	fn create_zone(
+		_node: Arc<Node>,
+		calling_client: Arc<Client>,
+		id: u64,
+		parent: Arc<Node>,
+		transform: Transform,
+		field: Arc<Node>,
+	) -> Result<()> {
+		let parent = parent.get_aspect::<Spatial>()?;
+		let transform = parse_transform(transform, true, true, false);
+		let field = field.get_aspect::<Field>()?;
+
+		let node = Node::from_id(&calling_client, id, true).add_to_scenegraph()?;
+		let space = Spatial::add_to(&node, Some(parent.clone()), transform, false);
+		Zone::add_to(&node, space, field);
+		Ok(())
+	}
+
+	async fn import_spatial_ref(
+		_node: Arc<Node>,
+		calling_client: Arc<Client>,
+		uid: u64,
+	) -> Result<Arc<Node>> {
+		Ok(EXPORTED_SPATIALS
+			.lock()
+			.get(&uid)
+			.and_then(|s| s.upgrade())
+			.map(|s| {
+				Alias::create(
+					&s,
+					&calling_client,
+					SPATIAL_REF_ASPECT_ALIAS_INFO.clone(),
+					None,
+				)
+				.unwrap()
+			})
+			.ok_or_eyre("Couldn't find spatial with that ID")?)
+	}
+}

src/nodes/spatial/zone.rs

@@ -0,0 +1,164 @@
+use super::{
+	SPATIAL_ASPECT_ALIAS_INFO, SPATIAL_REF_ASPECT_ALIAS_INFO, Spatial, ZONEABLE_REGISTRY,
+	ZoneAspect,
+};
+use crate::{
+	core::{client::Client, error::Result, registry::Registry},
+	nodes::{
+		Node,
+		alias::{Alias, AliasList, get_original},
+		fields::{Field, FieldTrait},
+	},
+};
+use glam::vec3a;
+use std::sync::{Arc, Weak};
+
+pub fn capture(spatial: &Arc<Spatial>, zone: &Arc<Zone>) {
+	let old_distance = spatial.zone_distance();
+	let new_distance = zone.field.distance(spatial, vec3a(0.0, 0.0, 0.0));
+	if new_distance.abs() > old_distance.abs() {
+		return;
+	}
+
+	release(spatial);
+	*spatial.old_parent.lock() = spatial.get_parent();
+	*spatial.zone.lock() = Arc::downgrade(zone);
+	let Some(zone_node) = zone.spatial.node.upgrade() else {
+		return;
+	};
+	let Some(spatial_node) = spatial.node.upgrade() else {
+		return;
+	};
+	let Ok(spatial_alias) = Alias::create(
+		&spatial_node,
+		&zone_node.get_client().unwrap(),
+		SPATIAL_ASPECT_ALIAS_INFO.clone(),
+		Some(&zone.captured),
+	) else {
+		return;
+	};
+	let _ = super::zone_client::capture(&zone_node, &spatial_alias);
+}
+pub fn release(spatial: &Spatial) {
+	let Some(spatial_node) = spatial.node.upgrade() else {
+		return;
+	};
+	let spatial = spatial_node.get_aspect::<Spatial>().unwrap();
+
+	let Some(old_parent) = spatial.old_parent.lock().take() else {
+		return;
+	};
+	let _ = spatial.set_spatial_parent_in_place(&old_parent);
+	let mut spatial_zone = spatial.zone.lock();
+
+	if let Some(spatial_zone) = spatial_zone.upgrade() {
+		spatial_zone.captured.remove_aspect(spatial.as_ref());
+		let Some(node) = spatial_zone.spatial.node.upgrade() else {
+			return;
+		};
+		let _ = super::zone_client::release(&node, spatial_node.id);
+	}
+	*spatial_zone = Weak::new();
+}
+
+pub struct Zone {
+	spatial: Arc<Spatial>,
+	pub field: Arc<Field>,
+	intersecting_spatials: Registry<Spatial>,
+	intersecting: AliasList,
+	captured: AliasList,
+}
+impl Zone {
+	pub fn add_to(node: &Arc<Node>, spatial: Arc<Spatial>, field: Arc<Field>) -> Arc<Zone> {
+		let zone = Arc::new(Zone {
+			spatial,
+			field,
+			intersecting_spatials: Registry::default(),
+			intersecting: AliasList::default(),
+			captured: AliasList::default(),
+		});
+		node.add_aspect_raw(zone.clone());
+		zone
+	}
+	pub fn update(&self) -> Result<()> {
+		let node = self.spatial.node().unwrap();
+
+		let current_zoneables = Registry::new();
+		for zoneable in ZONEABLE_REGISTRY.get_valid_contents() {
+			// Skip if the zoneable is an ancestor of the zone or the zone itself
+			if zoneable.is_ancestor_of(self.spatial.clone()) {
+				continue;
+			}
+
+			let distance = self.field.distance(&zoneable, [0.0; 3].into());
+			if distance > 0.0 {
+				continue;
+			}
+			if distance < zoneable.zone_distance() {
+				continue;
+			}
+			current_zoneables.add_raw(&zoneable);
+		}
+
+		let (added, removed) =
+			Registry::get_changes(&self.intersecting_spatials, &current_zoneables);
+		for added in added {
+			let Some(added_node) = added.node() else {
+				continue;
+			};
+			let Ok(alias) = Alias::create(
+				&added_node,
+				&self.spatial.node().unwrap().get_client().unwrap(),
+				SPATIAL_REF_ASPECT_ALIAS_INFO.clone(),
+				Some(&self.intersecting),
+			) else {
+				continue;
+			};
+			let _ = super::zone_client::enter(&node, &alias);
+		}
+		for removed in removed {
+			let Some(removed_node) = removed.node() else {
+				continue;
+			};
+			release(&removed);
+			let _ = super::zone_client::leave(&node, removed_node.id);
+			self.intersecting.remove_aspect(removed.as_ref());
+		}
+		self.intersecting_spatials.set(&current_zoneables);
+
+		Ok(())
+	}
+}
+impl ZoneAspect for Zone {
+	fn update(node: Arc<Node>, _calling_client: Arc<Client>) -> Result<()> {
+		let zone = node.get_aspect::<Zone>()?;
+		let _ = zone.update();
+		Ok(())
+	}
+
+	fn capture(node: Arc<Node>, _calling_client: Arc<Client>, spatial: Arc<Node>) -> Result<()> {
+		let zone = node.get_aspect::<Zone>()?;
+		let spatial = spatial.get_aspect()?;
+		capture(&spatial, &zone);
+		Ok(())
+	}
+
+	fn release(_node: Arc<Node>, _calling_client: Arc<Client>, spatial: Arc<Node>) -> Result<()> {
+		let spatial = spatial.get_aspect()?;
+		release(&spatial);
+		Ok(())
+	}
+}
+impl Drop for Zone {
+	fn drop(&mut self) {
+		for captured in self
+			.captured
+			.get_aliases()
+			.into_iter()
+			.filter_map(|n| get_original(n, false))
+			.filter_map(|n| n.get_aspect::<Spatial>().ok())
+		{
+			release(&captured);
+		}
+	}
+}

src/nodes/startup.rs

@@ -1,76 +0,0 @@
-use crate::core::client::Client;
-
-use super::Node;
-use anyhow::{anyhow, Result};
-use glam::Mat4;
-use parking_lot::Mutex;
-use rustc_hash::FxHashMap;
-use std::sync::Arc;
-
-lazy_static::lazy_static! {
-	pub static ref DESKTOP_STARTUP_IDS: Mutex<FxHashMap<String, StartupSettings>> = Default::default();
-}
-
-#[derive(Debug, Default, Clone)]
-pub struct StartupSettings {
-	pub transform: Mat4,
-}
-impl StartupSettings {
-	pub fn add_to(node: &Arc<Node>) {
-		node.startup_settings
-			.set(Mutex::new(StartupSettings::default()))
-			.unwrap();
-	}
-
-	fn set_root_flex(node: &Node, calling_client: Arc<Client>, data: &[u8]) -> Result<()> {
-		let startup_id = flexbuffers::Reader::get_root(data)?.get_str()?;
-		let spatial_node = calling_client
-			.scenegraph
-			.get_node(startup_id)
-			.ok_or_else(|| anyhow!("Root spatial node does not exist"))?;
-		let spatial = spatial_node
-			.spatial
-			.get()
-			.ok_or_else(|| anyhow!("Root spatial node is not a spatial"))?;
-		node.startup_settings.get().unwrap().lock().transform = spatial.global_transform();
-
-		Ok(())
-	}
-
-	fn generate_desktop_startup_id_flex(
-		node: &Node,
-		_calling_client: Arc<Client>,
-		_data: &[u8],
-	) -> Result<Vec<u8>> {
-		let id = nanoid::nanoid!();
-		let data = flexbuffers::singleton(id.as_str());
-		DESKTOP_STARTUP_IDS
-			.lock()
-			.insert(id, node.startup_settings.get().unwrap().lock().clone());
-		Ok(data)
-	}
-}
-
-pub fn create_interface(client: &Arc<Client>) {
-	let node = Node::create(client, "", "startup", false);
-	node.add_local_signal("createStartupSettings", create_startup_settings_flex);
-	node.add_to_scenegraph();
-}
-
-pub fn create_startup_settings_flex(
-	_node: &Node,
-	calling_client: Arc<Client>,
-	data: &[u8],
-) -> Result<()> {
-	let name = flexbuffers::Reader::get_root(data)?.get_str()?;
-	let node = Node::create(&calling_client, "/startup/settings", name, true).add_to_scenegraph();
-	StartupSettings::add_to(&node);
-
-	node.add_local_signal("setRoot", StartupSettings::set_root_flex);
-	node.add_local_method(
-		"generateDesktopStartupID",
-		StartupSettings::generate_desktop_startup_id_flex,
-	);
-
-	Ok(())
-}

src/objects/hmd.rs

@@ -0,0 +1,108 @@
+use std::sync::Arc;
+
+use bevy::prelude::*;
+use bevy_mod_openxr::{
+	helper_traits::{ToQuat as _, ToVec3 as _},
+	resources::OxrFrameState,
+	session::OxrSession,
+};
+use bevy_mod_xr::{
+	session::{XrPreDestroySession, XrSessionCreated, session_running},
+	spaces::{XrPrimaryReferenceSpace, XrSpace},
+};
+use openxr::SpaceLocationFlags;
+
+use crate::{DbusConnection, PreFrameWait, nodes::spatial::Spatial};
+
+use super::{ObjectHandle, SpatialRef, input::mouse_pointer::FlatscreenCam};
+
+pub struct HmdPlugin;
+impl Plugin for HmdPlugin {
+	fn build(&self, app: &mut App) {
+		app.add_systems(XrPreDestroySession, destroy_view_space);
+		app.add_systems(XrSessionCreated, create_view_space);
+		app.add_systems(PreFrameWait, update_xr.run_if(session_running));
+		app.add_systems(PreFrameWait, update_flat.run_if(not(session_running)));
+		app.add_systems(Startup, setup);
+	}
+}
+
+fn setup(connection: Res<DbusConnection>, mut cmds: Commands) {
+	let (spatial, _spatial_handle) = SpatialRef::create(&connection, "/org/stardustxr/HMD");
+	let hmd = Hmd {
+		spatial,
+		_spatial_handle,
+		space: None,
+	};
+	cmds.insert_resource(hmd);
+}
+
+fn create_view_space(session: Res<OxrSession>, mut hmd: ResMut<Hmd>) {
+	let space = session
+		.create_reference_space(openxr::ReferenceSpaceType::VIEW, Transform::IDENTITY)
+		.inspect_err(|err| error!("failed to create View XrSpace"))
+		.ok();
+	hmd.space = space.map(|v| v.0);
+}
+fn destroy_view_space(session: Res<OxrSession>, mut cmds: Commands, mut hmd: ResMut<Hmd>) {
+	let Some(space) = hmd.space.take() else {
+		return;
+	};
+	session.destroy_space(space);
+}
+
+#[derive(Resource)]
+struct Hmd {
+	spatial: Arc<Spatial>,
+	_spatial_handle: ObjectHandle<SpatialRef>,
+	space: Option<XrSpace>,
+}
+
+fn update_flat(cam: Single<&GlobalTransform, With<FlatscreenCam>>, hmd: Res<Hmd>) {
+	// this shouldn't be parented to anything, so global and local spaces should be the same
+	hmd.spatial.set_local_transform(cam.compute_matrix());
+}
+
+fn update_xr(
+	session: Option<Res<OxrSession>>,
+	ref_space: Option<Res<XrPrimaryReferenceSpace>>,
+	hmd: Res<Hmd>,
+	state: Option<Res<OxrFrameState>>,
+) {
+	let (Some(session), Some(view), Some(ref_space), Some(state)) =
+		(session, hmd.space, ref_space, state)
+	else {
+		// tokio::task::spawn({
+		// 	let handle = hmd.tracked_handle.clone();
+		// 	async move {
+		// 		handle.set_tracked(false);
+		// 	}
+		// });
+		return;
+	};
+	// this won't be correct with pipelined rendering
+	let location = session
+		.locate_space(&view, &ref_space, state.predicted_display_time)
+		.inspect_err(|err| error!("Error while Locating OpenXR Stage Space {err}"));
+	if let Ok(location) = location {
+		let is_tracked = location
+			.location_flags
+			.contains(SpaceLocationFlags::POSITION_VALID | SpaceLocationFlags::POSITION_TRACKED)
+			|| location.location_flags.contains(
+				SpaceLocationFlags::ORIENTATION_VALID | SpaceLocationFlags::ORIENTATION_TRACKED,
+			);
+		// tokio::task::spawn({
+		// 	let handle = play_space.tracked_handle.clone();
+		// 	async move {
+		// 		handle.set_tracked(is_tracked);
+		// 	}
+		// });
+		if is_tracked {
+			hmd.spatial
+				.set_local_transform(Mat4::from_rotation_translation(
+					location.pose.orientation.to_quat(),
+					location.pose.position.to_vec3(),
+				));
+		}
+	}
+}

src/objects/input/eye_pointer.rs

@@ -0,0 +1,51 @@
+use crate::{
+	core::client::INTERNAL_CLIENT,
+	nodes::{
+		Node, OwnedNode,
+		fields::{FieldTrait, Ray},
+		input::{INPUT_HANDLER_REGISTRY, InputDataType, InputMethod, Pointer},
+		spatial::Spatial,
+	},
+};
+use color_eyre::eyre::Result;
+use glam::{Mat4, vec3};
+use serde::{Deserialize, Serialize};
+use stardust_xr::values::Datamap;
+use std::sync::Arc;
+
+#[derive(Default, Deserialize, Serialize)]
+pub struct EyeDatamap {
+	eye: u32,
+}
+
+#[derive(Debug, Clone, Serialize)]
+pub struct KeyboardEvent {
+	pub keyboard: String,
+	pub keymap: Option<String>,
+	pub keys_up: Option<Vec<u32>>,
+	pub keys_down: Option<Vec<u32>>,
+}
+
+pub struct EyePointer {
+	node: OwnedNode,
+	spatial: Arc<Spatial>,
+	pointer: Arc<InputMethod>,
+}
+impl EyePointer {
+	pub fn new() -> Result<Self> {
+		let node = Node::generate(&INTERNAL_CLIENT, false).add_to_scenegraph_owned()?;
+		let spatial = Spatial::add_to(&node.0, None, Mat4::IDENTITY, false);
+		let pointer = InputMethod::add_to(
+			&node.0,
+			InputDataType::Pointer(Pointer::default()),
+			Datamap::from_typed(EyeDatamap::default())?,
+		)
+		.unwrap();
+
+		Ok(EyePointer {
+			node,
+			spatial,
+			pointer,
+		})
+	}
+}

src/objects/input/mod.rs

@@ -1,3 +1,93 @@
+pub mod eye_pointer;
 pub mod mouse_pointer;
-pub mod sk_controller;
-pub mod sk_hand;
+pub mod oxr_controller;
+pub mod oxr_hand;
+
+use crate::nodes::{
+	fields::{Field, FieldTrait, Ray},
+	input::{INPUT_HANDLER_REGISTRY, InputDataTrait, InputDataType, InputHandler, InputMethod},
+	spatial::Spatial,
+};
+use glam::vec3;
+use std::{
+	collections::VecDeque,
+	sync::{Arc, Weak},
+};
+
+#[derive(Default)]
+pub struct CaptureManager {
+	pub capture: Weak<InputHandler>,
+}
+impl CaptureManager {
+	pub fn update_capture(&mut self, method: &InputMethod) {
+		if let Some(capture) = &self.capture.upgrade() {
+			if !method
+				.capture_attempts
+				.get_valid_contents()
+				.contains(capture)
+			{
+				self.capture = Weak::new();
+			}
+		}
+	}
+	pub fn set_new_capture(
+		&mut self,
+		method: &InputMethod,
+		distance_calculator: DistanceCalculator,
+	) {
+		if self.capture.upgrade().is_none() {
+			self.capture = find_closest_capture(method, distance_calculator);
+		}
+	}
+	pub fn apply_capture(&self, method: &InputMethod) {
+		method.captures.clear();
+		if let Some(capture) = &self.capture.upgrade() {
+			method.set_handler_order([capture].into_iter());
+			method.captures.add_raw(capture);
+		}
+	}
+}
+
+type DistanceCalculator = fn(&Arc<Spatial>, &InputDataType, &Field) -> Option<f32>;
+
+pub fn find_closest_capture(
+	method: &InputMethod,
+	distance_calculator: DistanceCalculator,
+) -> Weak<InputHandler> {
+	method
+		.capture_attempts
+		.get_valid_contents()
+		.into_iter()
+		.filter_map(|h| {
+			distance_calculator(&method.spatial, &method.data.lock(), &h.field)
+				.map(|dist| (h.clone(), dist))
+		})
+		.min_by(|(_, dist_a), (_, dist_b)| dist_a.partial_cmp(dist_b).unwrap())
+		.map(|(handler, _)| Arc::downgrade(&handler))
+		.unwrap_or_default()
+}
+
+/// sorts them greatest to least distance (so you can pop off the closest ones easily)
+pub fn get_sorted_handlers(
+	method: &InputMethod,
+	distance_calculator: DistanceCalculator,
+) -> Vec<(Arc<InputHandler>, f32)> {
+	let mut handlers = INPUT_HANDLER_REGISTRY
+		.get_valid_contents()
+		.into_iter()
+		.filter(|handler| handler.spatial.node().is_some_and(|node| node.enabled()))
+		.filter(|handler| {
+			handler
+				.field
+				.spatial
+				.node()
+				.is_some_and(|node| node.enabled())
+		})
+		.filter_map(|handler| {
+			distance_calculator(&method.spatial, &method.data.lock(), &handler.field)
+				.map(|distance| (handler, distance))
+		})
+		.collect::<Vec<_>>();
+	handlers.sort_by(|(_, dist_a), (_, dist_b)| dist_a.partial_cmp(dist_b).unwrap());
+	handlers
+}

src/objects/input/mouse_pointer.rs

@@ -1,59 +1,482 @@
-use crate::nodes::{
-	input::{pointer::Pointer, InputMethod, InputType},
-	spatial::Spatial,
+use super::{CaptureManager, DistanceCalculator, get_sorted_handlers};
+use crate::{
+	DbusConnection, ObjectRegistryRes,
+	core::client::INTERNAL_CLIENT,
+	nodes::{
+		Node, OwnedNode,
+		fields::{EXPORTED_FIELDS, Field, FieldTrait, Ray},
+		input::{InputDataType, InputMethod, Pointer},
+		items::panel::KEYMAPS,
+		spatial::Spatial,
+	},
 };
-use glam::{vec3, Mat4};
-use stardust_xr::{schemas::flat::Datamap, values::Transform};
-use std::sync::{Arc, Weak};
-use stereokit::{
-	input::{ButtonState, Key, Ray},
-	StereoKit,
+use bevy::{
+	input::{
+		ButtonState,
+		keyboard::{KeyboardInput, NativeKey, NativeKeyCode},
+		mouse::{MouseMotion, MouseWheel},
+	},
+	prelude::*,
+	window::PrimaryWindow,
 };
+use color_eyre::eyre::Result;
+use glam::{Mat4, Vec3, vec3};
+use mint::Vector2;
+use serde::{Deserialize, Serialize};
+use slotmap::{DefaultKey, Key as SlotKey};
+use stardust_xr::{
+	schemas::dbus::{interfaces::FieldRefProxy, object_registry::ObjectRegistry},
+	values::Datamap,
+};
+use std::sync::Arc;
+use tokio::task::JoinSet;
+use tokio::time::{Duration, timeout};
+use xkbcommon_rs::{Context, Keymap, KeymapFormat, xkb_keymap::CompileFlags};
+use zbus::{Connection, names::OwnedInterfaceName};
 
+pub struct FlatscreenInputPlugin;
+
+impl Plugin for FlatscreenInputPlugin {
+	fn build(&self, app: &mut App) {
+		app.add_systems(Startup, setup);
+		// yes the input method will be delayed by one frame, its only for debugging anyways
+		app.add_systems(Update, update_pointer);
+	}
+}
+
+#[derive(Component)]
+#[require(Camera3d)]
+pub struct FlatscreenCam;
+
+fn setup(mut cmds: Commands) {
+	let Ok(pointer) =
+		MousePointer::new().inspect_err(|err| error!("unable to create mouse pointer: {err}"))
+	else {
+		return;
+	};
+	cmds.spawn((FlatscreenCam, Name::new("Flatscreen Camera")));
+	cmds.insert_resource(pointer);
+}
+
+fn update_pointer(
+	window: Single<(&Window), With<PrimaryWindow>>,
+	mut cam: Single<(&Camera, &GlobalTransform, &mut Transform), With<FlatscreenCam>>,
+	mut pointer: ResMut<MousePointer>,
+	connection: Res<DbusConnection>,
+	object_registry: Res<ObjectRegistryRes>,
+	mouse_buttons: Res<ButtonInput<MouseButton>>,
+	keyboard_buttons: Res<ButtonInput<KeyCode>>,
+	mut scroll: EventReader<MouseWheel>,
+	mut motion: EventReader<MouseMotion>,
+	mut keyboard_input_events: EventReader<KeyboardInput>,
+	time: Res<Time>,
+) {
+	let (cam, cam_transform, mut cam_local_transform) = cam.into_inner();
+	if keyboard_buttons.pressed(KeyCode::ShiftLeft) && mouse_buttons.pressed(MouseButton::Right) {
+		let (mut yaw, mut pitch, _) = cam_local_transform.rotation.to_euler(EulerRot::YXZ);
+
+		for e in motion.read() {
+			let scale = -0.003;
+			pitch += e.delta.y * scale;
+			yaw += e.delta.x * scale;
+		}
+
+		cam_local_transform.rotation = Quat::from_rotation_y(yaw) * Quat::from_rotation_x(pitch);
+
+		let mut move_vec = Vec3::ZERO;
+		move_vec.x += keyboard_buttons.pressed(KeyCode::KeyD) as u32 as f32;
+		move_vec.x -= keyboard_buttons.pressed(KeyCode::KeyA) as u32 as f32;
+		move_vec.z += keyboard_buttons.pressed(KeyCode::KeyS) as u32 as f32;
+		move_vec.z -= keyboard_buttons.pressed(KeyCode::KeyW) as u32 as f32;
+		move_vec.y += keyboard_buttons.pressed(KeyCode::KeyE) as u32 as f32;
+		move_vec.y -= keyboard_buttons.pressed(KeyCode::KeyQ) as u32 as f32;
+
+		let move_vec = cam_local_transform.rotation * move_vec.normalize_or_zero();
+		cam_local_transform.translation += move_vec * time.delta_secs() * 3.0;
+
+		return;
+	}
+	let Some(ray) = window
+		.cursor_position()
+		.and_then(|pos| get_viewport_pos(pos, cam))
+		.and_then(|pos| cam.viewport_to_world(cam_transform, pos).ok())
+	else {
+		return;
+	};
+	pointer.update(
+		&connection,
+		&object_registry,
+		ray,
+		&mouse_buttons,
+		&keyboard_buttons,
+		scroll,
+		keyboard_input_events,
+	);
+}
+
+fn get_viewport_pos(logical_pos: Vec2, cam: &Camera) -> Option<Vec2> {
+	if let Some(viewport_rect) = cam.logical_viewport_rect() {
+		if !viewport_rect.contains(logical_pos) {
+			return None;
+		}
+		Some(logical_pos - viewport_rect.min)
+	} else {
+		Some(logical_pos)
+	}
+}
+
+#[derive(Debug, Deserialize, Serialize)]
+struct MouseEvent {
+	select: f32,
+	middle: f32,
+	context: f32,
+	grab: f32,
+	scroll_continuous: Vector2<f32>,
+	scroll_discrete: Vector2<f32>,
+	raw_input_events: Vec<u32>,
+}
+impl Default for MouseEvent {
+	fn default() -> Self {
+		MouseEvent {
+			select: 0.0,
+			middle: 0.0,
+			context: 0.0,
+			grab: 0.0,
+			scroll_continuous: [0.0; 2].into(),
+			scroll_discrete: [0.0; 2].into(),
+			raw_input_events: vec![],
+		}
+	}
+}
+
+#[zbus::proxy(
+	interface = "org.stardustxr.XKBv1",
+	default_service = "org.stardustxr.XKBv1"
+)]
+trait KeyboardHandler {
+	async fn keymap(&self, keymap_id: u64) -> zbus::Result<()>;
+	async fn key_state(&self, key: u32, pressed: bool) -> zbus::Result<()>;
+	async fn reset(&self) -> zbus::Result<()>;
+}
+
+#[derive(Resource)]
 pub struct MousePointer {
+	node: OwnedNode,
+	keymap: DefaultKey,
+	spatial: Arc<Spatial>,
 	pointer: Arc<InputMethod>,
+	capture_manager: CaptureManager,
+	mouse_datamap: MouseEvent,
 }
 impl MousePointer {
-	pub fn new() -> Self {
-		MousePointer {
-			pointer: InputMethod::new(
-				Spatial::new(Weak::new(), None, Mat4::IDENTITY),
-				InputType::Pointer(Pointer::default()),
-			),
-		}
+	pub fn new() -> Result<Self> {
+		let node = Node::generate(&INTERNAL_CLIENT, false).add_to_scenegraph_owned()?;
+		let spatial = Spatial::add_to(&node.0, None, Mat4::IDENTITY, false);
+		let pointer = InputMethod::add_to(
+			&node.0,
+			InputDataType::Pointer(Pointer::default()),
+			Datamap::from_typed(MouseEvent::default())?,
+		)?;
+
+		let context = Context::new(0).unwrap();
+		let keymap = KEYMAPS.lock().insert(
+			Keymap::new_from_names(context, None, CompileFlags::NO_FLAGS)
+				.unwrap()
+				.get_as_string(KeymapFormat::TextV1)
+				.unwrap(),
+		);
+
+		Ok(MousePointer {
+			node,
+			spatial,
+			pointer,
+			capture_manager: CaptureManager::default(),
+			mouse_datamap: Default::default(),
+			keymap,
+		})
 	}
-	pub fn update(&self, sk: &StereoKit) {
-		if let Some(ray) = Ray::from_mouse(sk.input_mouse()) {
-			self.pointer.spatial.set_local_transform_components(
-				None,
-				Transform {
-					position: Some(ray.pos),
-					rotation: Some(
-						glam::Quat::from_rotation_arc(vec3(0.0, 0.0, 1.0), ray.dir.into()).into(),
-					),
-					scale: None,
-				},
-			);
+	pub fn update(
+		&mut self,
+		dbus_connection: &Connection,
+		object_registry: &ObjectRegistry,
+		ray: Ray3d,
+		mouse_buttons: &ButtonInput<MouseButton>,
+		keyboard_buttons: &ButtonInput<KeyCode>,
+		mut scroll: EventReader<MouseWheel>,
+		mut keyboard_input_events: EventReader<KeyboardInput>,
+	) {
+		let mut discrete = Vec2::ZERO;
+		let mut continuous = Vec2::ZERO;
+		for e in scroll.read() {
+			match e.unit {
+				bevy::input::mouse::MouseScrollUnit::Line => {
+					discrete.x += e.x;
+					discrete.y += e.y;
+				}
+				bevy::input::mouse::MouseScrollUnit::Pixel => {
+					continuous.x += e.x;
+					continuous.y += e.y;
+				}
+			}
 		}
-		let mut fbb = flexbuffers::Builder::default();
-		let mut map = fbb.start_map();
-		map.push(
-			"select",
-			if sk.input_key(Key::MouseLeft).contains(ButtonState::Active) {
-				1.0f32
-			} else {
-				0.0f32
-			},
+
+		self.spatial.set_local_transform(
+			Mat4::look_to_rh(ray.origin, Vec3::from(ray.direction), Vec3::Y).inverse(),
 		);
-		map.push(
-			"grab",
-			if sk.input_key(Key::MouseRight).contains(ButtonState::Active) {
-				1.0f32
-			} else {
-				0.0f32
-			},
+		{
+			// Set pointer input datamap
+			self.mouse_datamap = MouseEvent {
+				select: mouse_buttons.pressed(MouseButton::Left) as u32 as f32,
+				middle: mouse_buttons.pressed(MouseButton::Middle) as u32 as f32,
+				context: mouse_buttons.pressed(MouseButton::Right) as u32 as f32,
+				grab: mouse_buttons.pressed(MouseButton::Right) as u32 as f32, // Was Mouse 5
+				scroll_continuous: continuous.into(),
+				scroll_discrete: discrete.into(),
+				raw_input_events: mouse_buttons
+					.get_pressed()
+					.map(|button| match button {
+						MouseButton::Left => input_event_codes::BTN_LEFT!(),
+						MouseButton::Right => input_event_codes::BTN_RIGHT!(),
+						MouseButton::Middle => input_event_codes::BTN_MIDDLE!(),
+						MouseButton::Back => input_event_codes::BTN_BACK!(),
+						MouseButton::Forward => input_event_codes::BTN_FORWARD!(),
+						MouseButton::Other(b) => *b as u32,
+					})
+					.collect(),
+			};
+			*self.pointer.datamap.lock() = Datamap::from_typed(&self.mouse_datamap).unwrap();
+		}
+		self.target_pointer_input();
+		self.send_keyboard_input(dbus_connection, object_registry, keyboard_input_events);
+	}
+	fn target_pointer_input(&mut self) {
+		let distance_calculator: DistanceCalculator = |space, data, field| {
+			let result = field.ray_march(Ray {
+				origin: vec3(0.0, 0.0, 0.0),
+				direction: vec3(0.0, 0.0, -1.0),
+				space: space.clone(),
+			});
+			let valid =
+				result.deepest_point_distance > 0.0 && result.min_distance.is_sign_negative();
+			valid.then_some(result.deepest_point_distance)
+		};
+
+		self.capture_manager.update_capture(&self.pointer);
+		self.capture_manager
+			.set_new_capture(&self.pointer, distance_calculator);
+		self.capture_manager.apply_capture(&self.pointer);
+
+		if self.capture_manager.capture.upgrade().is_some() {
+			return;
+		}
+
+		let mut handlers = get_sorted_handlers(&self.pointer, distance_calculator);
+		let first_distance = handlers
+			.first()
+			.map(|(_, distance)| *distance)
+			.unwrap_or(f32::NEG_INFINITY);
+
+		self.pointer.set_handler_order(
+			handlers
+				.iter()
+				.filter(|(handler, distance)| (distance - first_distance).abs() <= 0.001)
+				.map(|(handler, _)| handler),
 		);
-		map.end_map();
-		*self.pointer.datamap.lock() = Datamap::new(fbb.take_buffer()).ok();
+	}
+
+	pub fn send_keyboard_input(
+		&mut self,
+		dbus_connection: &Connection,
+		object_registry: &ObjectRegistry,
+		mut keyboard_input_events: EventReader<KeyboardInput>,
+	) {
+		let keyboard_handlers = object_registry.get_objects("org.stardustxr.XKBv1");
+		let events = keyboard_input_events
+			.read()
+			.filter_map(|e| Some((map_key(e.key_code)?, e.state)))
+			.collect::<Vec<_>>();
+		// Spawn async task to handle keyboard input
+		tokio::spawn({
+			let keyboard_handlers = keyboard_handlers.clone();
+			let spatial = self.spatial.clone();
+			let keymap_id = self.keymap.data().as_ffi();
+			let dbus_connection = dbus_connection.clone();
+
+			async move {
+				let mut closest_handler = None;
+				let mut closest_distance = f32::MAX;
+
+				let mut join_set = JoinSet::new();
+				for handler in &keyboard_handlers {
+					let handler = handler.clone();
+					let dbus_connection = dbus_connection.clone();
+					join_set.spawn(async move {
+						// TODO: refactor the whole thing so picking the keyboardhandler to send input to is separate from sending
+						timeout(Duration::from_millis(10), async {
+							let field_ref = handler
+								.to_typed_proxy::<FieldRefProxy>(&dbus_connection)
+								.await
+								.ok()?;
+							let uid = field_ref.uid().await.ok()?;
+							Some((handler, uid))
+						})
+						.await
+						.ok()
+						.flatten()
+					});
+				}
+				while let Some(Ok(Some((handler, field_ref_id)))) = join_set.join_next().await {
+					let exported_fields = EXPORTED_FIELDS.lock();
+					let Some(field_ref_node) =
+						exported_fields.get(&field_ref_id).and_then(|f| f.upgrade())
+					else {
+						println!("didn't find a thing :(");
+						continue;
+					};
+					// println!("still sendin stuff :)");
+					let Ok(field_ref) = field_ref_node.get_aspect::<Field>() else {
+						continue;
+					};
+					drop(exported_fields);
+
+					let result = field_ref.ray_march(Ray {
+						origin: vec3(0.0, 0.0, 0.0),
+						direction: vec3(0.0, 0.0, -1.0),
+						space: spatial.clone(),
+					});
+
+					if result.deepest_point_distance > 0.0
+						&& result.min_distance < 0.05
+						&& result.deepest_point_distance < closest_distance
+					{
+						closest_distance = result.deepest_point_distance;
+						closest_handler = Some(handler);
+					}
+				}
+
+				let Some(handler) = closest_handler else {
+					return;
+				};
+				let Ok(keyboard_handler) = handler
+					.to_typed_proxy::<KeyboardHandlerProxy>(&dbus_connection)
+					.await
+				else {
+					return;
+				};
+
+				// Register keymap first
+				let _ = keyboard_handler.keymap(keymap_id).await;
+
+				// Send key states
+				for (key, state) in events.iter() {
+					let pressed = matches!(state, ButtonState::Pressed);
+					let _ = keyboard_handler.key_state(key + 8, pressed).await;
+				}
+			}
+		});
+	}
+}
+
+fn map_key(key: KeyCode) -> Option<u32> {
+	use KeyCode as Key;
+	match key {
+		Key::Unidentified(NativeKeyCode::Xkb(code)) => Some(code),
+		Key::Backspace => Some(input_event_codes::KEY_BACKSPACE!()),
+		Key::Tab => Some(input_event_codes::KEY_TAB!()),
+		Key::Enter => Some(input_event_codes::KEY_ENTER!()),
+		Key::ShiftLeft => Some(input_event_codes::KEY_LEFTSHIFT!()),
+		Key::ControlLeft => Some(input_event_codes::KEY_LEFTCTRL!()),
+		Key::AltLeft => Some(input_event_codes::KEY_LEFTALT!()),
+		Key::CapsLock => Some(input_event_codes::KEY_CAPSLOCK!()),
+		Key::Escape => Some(input_event_codes::KEY_ESC!()),
+		Key::Space => Some(input_event_codes::KEY_SPACE!()),
+		Key::End => Some(input_event_codes::KEY_END!()),
+		Key::Home => Some(input_event_codes::KEY_HOME!()),
+		Key::ArrowLeft => Some(input_event_codes::KEY_LEFT!()),
+		Key::ArrowRight => Some(input_event_codes::KEY_RIGHT!()),
+		Key::ArrowUp => Some(input_event_codes::KEY_UP!()),
+		Key::ArrowDown => Some(input_event_codes::KEY_DOWN!()),
+		Key::PageUp => Some(input_event_codes::KEY_PAGEUP!()),
+		Key::PageDown => Some(input_event_codes::KEY_PAGEDOWN!()),
+		Key::PrintScreen => Some(input_event_codes::KEY_PRINT!()),
+		Key::Insert => Some(input_event_codes::KEY_INSERT!()),
+		Key::Delete => Some(input_event_codes::KEY_DELETE!()),
+		Key::Digit0 => Some(input_event_codes::KEY_0!()),
+		Key::Digit1 => Some(input_event_codes::KEY_1!()),
+		Key::Digit2 => Some(input_event_codes::KEY_2!()),
+		Key::Digit3 => Some(input_event_codes::KEY_3!()),
+		Key::Digit4 => Some(input_event_codes::KEY_4!()),
+		Key::Digit5 => Some(input_event_codes::KEY_5!()),
+		Key::Digit6 => Some(input_event_codes::KEY_6!()),
+		Key::Digit7 => Some(input_event_codes::KEY_7!()),
+		Key::Digit8 => Some(input_event_codes::KEY_8!()),
+		Key::Digit9 => Some(input_event_codes::KEY_9!()),
+		Key::KeyA => Some(input_event_codes::KEY_A!()),
+		Key::KeyB => Some(input_event_codes::KEY_B!()),
+		Key::KeyC => Some(input_event_codes::KEY_C!()),
+		Key::KeyD => Some(input_event_codes::KEY_D!()),
+		Key::KeyE => Some(input_event_codes::KEY_E!()),
+		Key::KeyF => Some(input_event_codes::KEY_F!()),
+		Key::KeyG => Some(input_event_codes::KEY_G!()),
+		Key::KeyH => Some(input_event_codes::KEY_H!()),
+		Key::KeyI => Some(input_event_codes::KEY_I!()),
+		Key::KeyJ => Some(input_event_codes::KEY_J!()),
+		Key::KeyK => Some(input_event_codes::KEY_K!()),
+		Key::KeyL => Some(input_event_codes::KEY_L!()),
+		Key::KeyM => Some(input_event_codes::KEY_M!()),
+		Key::KeyN => Some(input_event_codes::KEY_N!()),
+		Key::KeyO => Some(input_event_codes::KEY_O!()),
+		Key::KeyP => Some(input_event_codes::KEY_P!()),
+		Key::KeyQ => Some(input_event_codes::KEY_Q!()),
+		Key::KeyR => Some(input_event_codes::KEY_R!()),
+		Key::KeyS => Some(input_event_codes::KEY_S!()),
+		Key::KeyT => Some(input_event_codes::KEY_T!()),
+		Key::KeyU => Some(input_event_codes::KEY_U!()),
+		Key::KeyV => Some(input_event_codes::KEY_V!()),
+		Key::KeyW => Some(input_event_codes::KEY_W!()),
+		Key::KeyX => Some(input_event_codes::KEY_X!()),
+		Key::KeyY => Some(input_event_codes::KEY_Y!()),
+		Key::KeyZ => Some(input_event_codes::KEY_Z!()),
+		Key::Numpad0 => Some(input_event_codes::KEY_NUMERIC_0!()),
+		Key::Numpad1 => Some(input_event_codes::KEY_NUMERIC_1!()),
+		Key::Numpad2 => Some(input_event_codes::KEY_NUMERIC_2!()),
+		Key::Numpad3 => Some(input_event_codes::KEY_NUMERIC_3!()),
+		Key::Numpad4 => Some(input_event_codes::KEY_NUMERIC_4!()),
+		Key::Numpad5 => Some(input_event_codes::KEY_NUMERIC_5!()),
+		Key::Numpad6 => Some(input_event_codes::KEY_NUMERIC_6!()),
+		Key::Numpad7 => Some(input_event_codes::KEY_NUMERIC_7!()),
+		Key::Numpad8 => Some(input_event_codes::KEY_NUMERIC_8!()),
+		Key::Numpad9 => Some(input_event_codes::KEY_NUMERIC_9!()),
+		Key::F1 => Some(input_event_codes::KEY_F1!()),
+		Key::F2 => Some(input_event_codes::KEY_F2!()),
+		Key::F3 => Some(input_event_codes::KEY_F3!()),
+		Key::F4 => Some(input_event_codes::KEY_F4!()),
+		Key::F5 => Some(input_event_codes::KEY_F5!()),
+		// Key::F6 => Some(input_event_codes::KEY_F6!()),
+		// Key::F7 => Some(input_event_codes::KEY_F7!()),
+		// Key::F8 => Some(input_event_codes::KEY_F8!()),
+		Key::F9 => Some(input_event_codes::KEY_F9!()),
+		Key::F10 => Some(input_event_codes::KEY_F10!()),
+		Key::F11 => Some(input_event_codes::KEY_F11!()),
+		Key::F12 => Some(input_event_codes::KEY_F12!()),
+		Key::Comma => Some(input_event_codes::KEY_COMMA!()),
+		Key::Period => Some(input_event_codes::KEY_DOT!()),
+		Key::Slash => Some(input_event_codes::KEY_SLASH!()),
+		Key::Backslash => Some(input_event_codes::KEY_BACKSLASH!()),
+		Key::Semicolon => Some(input_event_codes::KEY_SEMICOLON!()),
+		Key::Quote => Some(input_event_codes::KEY_APOSTROPHE!()),
+		Key::BracketLeft => Some(input_event_codes::KEY_LEFTBRACE!()),
+		Key::BracketRight => Some(input_event_codes::KEY_RIGHTBRACE!()),
+		Key::Minus => Some(input_event_codes::KEY_MINUS!()),
+		Key::Equal => Some(input_event_codes::KEY_EQUAL!()),
+		Key::Backquote => Some(input_event_codes::KEY_GRAVE!()),
+		Key::SuperLeft => Some(input_event_codes::KEY_LEFTMETA!()),
+		Key::SuperRight => Some(input_event_codes::KEY_RIGHTMETA!()),
+		Key::NumpadMultiply => Some(input_event_codes::KEY_NUMERIC_STAR!()),
+		Key::NumpadAdd => Some(input_event_codes::KEY_KPPLUS!()),
+		Key::NumpadSubtract => Some(input_event_codes::KEY_MINUS!()),
+		Key::NumpadDecimal => Some(input_event_codes::KEY_DOT!()),
+		Key::NumpadDivide => Some(input_event_codes::KEY_SLASH!()),
+		_ => None,
 	}
 }

src/objects/input/oxr_controller.rs

@@ -0,0 +1,414 @@
+use super::{CaptureManager, get_sorted_handlers};
+use crate::{
+	DbusConnection, PreFrameWait,
+	core::client::INTERNAL_CLIENT,
+	nodes::{
+		Node, OwnedNode,
+		drawable::{
+			MaterialParameter,
+			model::{Model, ModelPart},
+		},
+		fields::{Field, FieldTrait},
+		input::{INPUT_HANDLER_REGISTRY, InputDataType, InputHandler, InputMethod, Tip},
+		spatial::Spatial,
+	},
+	objects::{ObjectHandle, SpatialRef, Tracked},
+};
+use bevy::{asset::Handle, ecs::resource::Resource};
+use bevy::{math::Affine3, prelude::*};
+use bevy_mod_openxr::{
+	action_binding::{OxrSendActionBindings, OxrSuggestActionBinding},
+	helper_traits::{ToIsometry3d, ToVec2},
+	resources::{OxrFrameState, OxrInstance},
+	session::OxrSession,
+};
+use bevy_mod_xr::{
+	hands::HandSide,
+	session::{XrPreDestroySession, XrSessionCreated, XrSessionCreatedEvent},
+	spaces::{XrPrimaryReferenceSpace, XrReferenceSpace, XrSpace},
+};
+use color_eyre::eyre::Result;
+use glam::{Affine3A, Mat4, Vec2, Vec3};
+use openxr::{Action, ActiveActionSet, SpaceLocationFlags};
+use serde::{Deserialize, Serialize};
+use stardust_xr::values::{Datamap, ResourceID, color::Rgb};
+use std::{
+	borrow::Cow,
+	fs,
+	path::{Path, PathBuf},
+	str::FromStr,
+	sync::Arc,
+};
+use tracing::instrument;
+use zbus::Connection;
+pub struct ControllerPlugin;
+const CURSOR_MODEL_PATH: &str = "/tmp/stardust_server/models/cursor.glb";
+impl Plugin for ControllerPlugin {
+	fn build(&self, app: &mut App) {
+		let cursor = include_bytes!("cursor.glb");
+		fs::create_dir_all(
+			PathBuf::from_str(CURSOR_MODEL_PATH)
+				.unwrap()
+				.parent()
+				.unwrap(),
+		);
+		fs::write(CURSOR_MODEL_PATH, cursor).expect("can't write tmp cursor model file");
+		app.add_systems(OxrSendActionBindings, suggest_bindings.run_if(run_once));
+		app.add_systems(
+			PostUpdate,
+			create_spaces.run_if(on_event::<XrSessionCreatedEvent>),
+		);
+		app.add_systems(XrPreDestroySession, destroy_spaces);
+		app.add_systems(Startup, setup.run_if(resource_exists::<OxrInstance>));
+		app.add_systems(PreFrameWait, update.run_if(resource_exists::<Controllers>));
+	}
+}
+
+// the api is just slightly nicer when using the bevy_mod_openxr solution okay?
+fn suggest_bindings(
+	instance: Res<OxrInstance>,
+	actions: Res<Actions>,
+	mut suggest: EventWriter<OxrSuggestActionBinding>,
+) {
+	let mut bind_all = |interaction_profile: &'static str,
+	                    bindings: &[(openxr::sys::Action, &[&'static str])]| {
+		for (action, bindings) in bindings {
+			suggest.write(OxrSuggestActionBinding {
+				action: *action,
+				interaction_profile: interaction_profile.into(),
+				bindings: bindings.iter().copied().map(Cow::Borrowed).collect(),
+			});
+		}
+	};
+	bind_all(
+		"/interaction_profiles/oculus/touch_controller",
+		&[
+			(
+				actions.trigger.as_raw(),
+				&[
+					"/user/hand/left/input/trigger/value",
+					"/user/hand/right/input/trigger/value",
+				],
+			),
+			(
+				actions.stick_click.as_raw(),
+				&[
+					"/user/hand/left/input/thumbstick/click",
+					"/user/hand/right/input/thumbstick/click",
+				],
+			),
+			(
+				actions.button.as_raw(),
+				&[
+					"/user/hand/left/input/x/click",
+					"/user/hand/left/input/y/click",
+					"/user/hand/right/input/a/click",
+					"/user/hand/right/input/b/click",
+				],
+			),
+			(
+				actions.grip.as_raw(),
+				&[
+					"/user/hand/left/input/squeeze/value",
+					"/user/hand/right/input/squeeze/value",
+				],
+			),
+			(
+				actions.stick.as_raw(),
+				&[
+					"/user/hand/left/input/thumbstick",
+					"/user/hand/right/input/thumbstick",
+				],
+			),
+			(
+				actions.space.as_raw(),
+				&[
+					"/user/hand/left/input/aim/pose",
+					"/user/hand/right/input/aim/pose",
+				],
+			),
+		],
+	);
+	bind_all(
+		"/interaction_profiles/khr/simple_controller",
+		&[(
+			actions.space.as_raw(),
+			&[
+				"/user/hand/left/input/aim/pose",
+				"/user/hand/right/input/aim/pose",
+			],
+		)],
+	);
+}
+
+fn update(
+	mut controllers: ResMut<Controllers>,
+	actions: Res<Actions>,
+	session: Option<Res<OxrSession>>,
+	ref_space: Option<Res<XrPrimaryReferenceSpace>>,
+	state: Option<Res<OxrFrameState>>,
+) {
+	let (Some(session), Some(state), Some(ref_space)) = (session, state, ref_space) else {
+		controllers.left.set_enabled(false);
+		controllers.right.set_enabled(false);
+		return;
+	};
+	debug_span!("sync actions").in_scope(|| {
+		session
+			.sync_actions(&[ActiveActionSet::new(&actions.set)])
+			.unwrap();
+	});
+	let time = state.predicted_display_time;
+	// stupid bevy gltf loading issue (rotated 180 degrees on the y axis)
+	controllers
+		.left
+		.update(&session, &actions, time, ref_space.0);
+	controllers
+		.right
+		.update(&session, &actions, time, ref_space.0);
+}
+
+fn create_spaces(
+	session: Res<OxrSession>,
+	mut controllers: ResMut<Controllers>,
+	actions: Res<Actions>,
+) {
+	// if we ever need more actions than just these we should fully swith to the
+	// bevy_mod_openxr provided stuff
+	session.attach_action_sets(&[&actions.set]);
+	session
+		.sync_actions(&[ActiveActionSet::new(&actions.set)])
+		.unwrap();
+
+	let instance = session.instance();
+	let left = instance.string_to_path("/user/hand/left").unwrap();
+	let right = instance.string_to_path("/user/hand/right").unwrap();
+	let left = session
+		.create_action_space(&actions.space, left, Isometry3d::IDENTITY)
+		.unwrap();
+	let right = session
+		.create_action_space(&actions.space, right, Isometry3d::IDENTITY)
+		.unwrap();
+	controllers.left.space = Some(left);
+	controllers.right.space = Some(right);
+}
+
+fn destroy_spaces(session: Res<OxrSession>, mut controllers: ResMut<Controllers>) {
+	if let Some(space) = controllers.left.space.take() {
+		session.destroy_space(space);
+	}
+	if let Some(space) = controllers.right.space.take() {
+		session.destroy_space(space);
+	}
+}
+
+fn setup(instance: Res<OxrInstance>, connection: Res<DbusConnection>, mut cmds: Commands) {
+	tokio::task::spawn({
+		let connection = connection.clone();
+		async move {
+			connection
+				.request_name("org.stardustxr.Controllers")
+				.await
+				.unwrap();
+		}
+	});
+	let set = instance
+		.create_action_set("input_method_actions", "Input Method Action Source", 0)
+		.unwrap();
+	let paths = &[
+		instance.string_to_path("/user/hand/left").unwrap(),
+		instance.string_to_path("/user/hand/right").unwrap(),
+	];
+	let actions = Actions {
+		trigger: set.create_action("trigger", "Select", paths).unwrap(),
+		stick_click: set.create_action("stick_click", "Middle", paths).unwrap(),
+		button: set.create_action("face_button", "Context", paths).unwrap(),
+		grip: set.create_action("grip", "Grab", paths).unwrap(),
+		stick: set.create_action("stick", "Scroll", paths).unwrap(),
+		space: set.create_action("pose", "Location", paths).unwrap(),
+		set,
+	};
+	let controllers = Controllers {
+		left: OxrControllerInput::new(&connection, HandSide::Left).unwrap(),
+		right: OxrControllerInput::new(&connection, HandSide::Right).unwrap(),
+	};
+	cmds.insert_resource(controllers);
+	cmds.insert_resource(actions);
+}
+
+#[derive(Default, Debug, Deserialize, Serialize)]
+struct ControllerDatamap {
+	select: f32,
+	middle: f32,
+	context: f32,
+	grab: f32,
+	scroll: Vec2,
+}
+#[derive(Resource)]
+struct Actions {
+	set: openxr::ActionSet,
+	trigger: openxr::Action<f32>,
+	stick_click: openxr::Action<f32>,
+	button: openxr::Action<f32>,
+	grip: openxr::Action<f32>,
+	space: openxr::Action<openxr::Posef>,
+	stick: openxr::Action<openxr::Vector2f>,
+}
+#[derive(Resource)]
+struct Controllers {
+	left: OxrControllerInput,
+	right: OxrControllerInput,
+}
+
+pub struct OxrControllerInput {
+	object_handle: ObjectHandle<SpatialRef>,
+	input: Arc<InputMethod>,
+	side: HandSide,
+	model: Arc<Model>,
+	model_part: Arc<ModelPart>,
+	capture_manager: CaptureManager,
+	datamap: ControllerDatamap,
+	tracked: ObjectHandle<Tracked>,
+	space: Option<XrSpace>,
+}
+impl OxrControllerInput {
+	fn new(connection: &Connection, side: HandSide) -> Result<Self> {
+		let path = "/org/stardustxr/Controller/".to_string()
+			+ match side {
+				HandSide::Left => "left",
+				HandSide::Right => "right",
+			};
+		let (spatial, object_handle) = SpatialRef::create(connection, &path);
+		let tracked = Tracked::new(connection, &path);
+		let tip = InputDataType::Tip(Tip::default());
+		let node = spatial.node().unwrap();
+		node.set_enabled(false);
+		let model = Model::add_to(&node, ResourceID::Direct(CURSOR_MODEL_PATH.into())).unwrap();
+		let model_part = model.get_model_part("Cursor".to_string()).unwrap();
+		let input = InputMethod::add_to(
+			&node,
+			tip,
+			Datamap::from_typed(ControllerDatamap::default())?,
+		)?;
+		Ok(OxrControllerInput {
+			object_handle,
+			input,
+			side,
+			model,
+			model_part,
+			capture_manager: CaptureManager::default(),
+			datamap: Default::default(),
+			tracked,
+			space: None,
+		})
+	}
+	#[instrument(level = "debug", skip(self))]
+	pub fn set_enabled(&self, enabled: bool) {
+		if let Some(node) = self.input.spatial.node() {
+			node.set_enabled(enabled);
+		}
+		tokio::spawn({
+			// this is suboptimal since it probably allocates a fresh string every frame
+			let handle = self.tracked.clone();
+			async move {
+				handle.set_tracked(enabled).await;
+			}
+		});
+	}
+	fn update(
+		&mut self,
+		session: &OxrSession,
+		actions: &Actions,
+		time: openxr::Time,
+		ref_space: XrReferenceSpace,
+	) {
+		let Some(space) = self.space.as_ref() else {
+			return;
+		};
+		let _span = debug_span!("locate space").entered();
+		let Ok(location) = session
+			.locate_space(space, &ref_space, time)
+			.inspect_err(|err| error!("error while locating controller space: {err}"))
+		else {
+			return;
+		};
+		let enabled = location.location_flags.contains(
+			SpaceLocationFlags::POSITION_VALID
+				| SpaceLocationFlags::POSITION_TRACKED
+				| SpaceLocationFlags::ORIENTATION_VALID
+				| SpaceLocationFlags::ORIENTATION_TRACKED,
+		);
+		drop(_span);
+		self.set_enabled(enabled);
+		if enabled {
+			let world_transform = Mat4::from(Affine3A::from(location.pose.to_xr_pose()));
+			self.model_part
+				.set_material_parameter("roughness".to_string(), MaterialParameter::Float(1.0));
+			self.model_part.set_material_parameter(
+				"color".to_string(),
+				MaterialParameter::Color(stardust_xr::values::Color::new(
+					if self.capture_manager.capture.upgrade().is_none() {
+						Rgb::new(1.0, 1.0, 1.0)
+					} else {
+						Rgb::new(0.0, 1.0, 0.75)
+					},
+					1.0,
+				)),
+			);
+			self.input
+				.spatial
+				.set_local_transform(world_transform * Mat4::from_scale(Vec3::splat(0.02)));
+		}
+		let path = session
+			.instance()
+			.string_to_path(match self.side {
+				HandSide::Left => "/user/hand/left",
+				HandSide::Right => "/user/hand/right",
+			})
+			.unwrap();
+		if let Ok(path) = session.current_interaction_profile(path) {
+			if session.instance().path_to_string(path).unwrap()
+				== "/interaction_profiles/khr/simple_controller"
+			{
+				self.set_enabled(false);
+			}
+		}
+		fn get<T: openxr::ActionInput + Default>(
+			session: &OxrSession,
+			path: openxr::Path,
+			action: &Action<T>,
+		) -> T {
+			action
+				.state(session, path)
+				.map(|v| v.current_state)
+				.unwrap_or_default()
+		}
+		let _span = debug_span!("apply datamap").entered();
+		self.datamap = ControllerDatamap {
+			select: get(session, path, &actions.trigger),
+			middle: get(session, path, &actions.stick_click) as u32 as f32,
+			context: get(session, path, &actions.button) as u32 as f32,
+			grab: get(session, path, &actions.grip),
+			scroll: get(session, path, &actions.stick).to_vec2(),
+		};
+		*self.input.datamap.lock() = Datamap::from_typed(&self.datamap).unwrap();
+		drop(_span);
+
+		let distance_calculator = |space: &Arc<Spatial>, _data: &InputDataType, field: &Field| {
+			Some(field.distance(space, [0.0; 3].into()).abs())
+		};
+
+		self.capture_manager.update_capture(&self.input);
+		self.capture_manager
+			.set_new_capture(&self.input, distance_calculator);
+		self.capture_manager.apply_capture(&self.input);
+
+		if self.capture_manager.capture.upgrade().is_some() {
+			return;
+		}
+
+		let sorted_handlers = get_sorted_handlers(&self.input, distance_calculator);
+		self.input
+			.set_handler_order(sorted_handlers.iter().map(|(handler, _)| handler));
+	}
+}

src/objects/input/oxr_hand.rs

@@ -0,0 +1,371 @@
+use crate::core::client::INTERNAL_CLIENT;
+use crate::nodes::OwnedNode;
+use crate::nodes::fields::{Field, FieldTrait};
+use crate::nodes::input::{Finger, INPUT_HANDLER_REGISTRY, InputDataType, InputHandler, Thumb};
+use crate::nodes::{
+	Node,
+	input::{Hand, InputMethod, Joint},
+	spatial::Spatial,
+};
+use crate::objects::{ObjectHandle, SpatialRef, Tracked};
+use crate::{BevyMaterial, DbusConnection, ObjectRegistryRes, PreFrameWait};
+use bevy::prelude::Transform as BevyTransform;
+use bevy::prelude::*;
+use bevy_mod_openxr::helper_traits::{ToQuat, ToVec3};
+use bevy_mod_openxr::resources::OxrFrameState;
+use bevy_mod_openxr::session::OxrSession;
+use bevy_mod_xr::hands::{HandBone, HandSide, XrHandBoneEntities, XrHandBoneRadius};
+use bevy_mod_xr::session::{XrPreDestroySession, XrSessionCreated};
+use bevy_mod_xr::spaces::{XrPrimaryReferenceSpace, XrSpaceLocationFlags};
+use bevy_sk::hand::GRADIENT_TEXTURE_HANDLE;
+use color_eyre::eyre::Result;
+use glam::{Mat4, Quat, Vec3};
+use openxr::{HandJointLocation, SpaceLocationFlags};
+use serde::{Deserialize, Serialize};
+use stardust_xr::values::Datamap;
+use std::sync::Arc;
+use zbus::Connection;
+
+use super::{CaptureManager, get_sorted_handlers};
+
+pub struct HandPlugin;
+impl Plugin for HandPlugin {
+	fn build(&self, app: &mut App) {
+		app.add_systems(PreFrameWait, update_hands.run_if(resource_exists::<Hands>));
+		app.add_systems(XrSessionCreated, create_trackers);
+		app.add_systems(XrPreDestroySession, destroy_trackers);
+		app.add_systems(PostUpdate, update_hand_material);
+		app.add_systems(Startup, setup);
+	}
+}
+fn update_hands(
+	mut hands: ResMut<Hands>,
+	session: Option<Res<OxrSession>>,
+	state: Option<Res<OxrFrameState>>,
+	ref_space: Option<Res<XrPrimaryReferenceSpace>>,
+	mut materials: ResMut<Assets<BevyMaterial>>,
+	mut joint_query: Query<(
+		&mut BevyTransform,
+		&mut XrSpaceLocationFlags,
+		&mut XrHandBoneRadius,
+	)>,
+	joints_query: Query<&XrHandBoneEntities>,
+) {
+	let (Some(session), Some(state), Some(ref_space)) = (session, state, ref_space) else {
+		tokio::task::spawn({
+			let left = hands.left.tracked.clone();
+			let right = hands.right.tracked.clone();
+			async move {
+				left.set_tracked(false);
+				right.set_tracked(false);
+			}
+		});
+		return;
+	};
+	let get_joints = |hand: &mut OxrHandInput| -> Option<openxr::HandJointLocations> {
+		let Some(tracker) = hand.tracker.as_ref() else {
+			hand.input.spatial.node().unwrap().set_enabled(false);
+			let handle = hand.tracked.clone();
+			tokio::task::spawn(async move {
+				handle.set_tracked(false);
+			});
+			return None;
+		};
+		// this won't be correct with pipelined rendering
+		session
+			.locate_hand_joints(tracker, &ref_space, state.predicted_display_time)
+			.inspect_err(|err| error!("Error while locating hand joints"))
+			.ok()
+			.flatten()
+	};
+	let joints_left = get_joints(&mut hands.left);
+	let joints_right = get_joints(&mut hands.right);
+	hands.left.update(joints_left.as_ref(), &mut materials);
+	hands.right.update(joints_right.as_ref(), &mut materials);
+}
+
+fn pinch_between(joint_1: &Joint, joint_2: &Joint) -> f32 {
+	const PINCH_MAX: f32 = 0.11;
+	const PINCH_ACTIVACTION_DISTANCE: f32 = 0.01;
+	let combined_radius = joint_1.radius + joint_2.radius;
+	let pinch_dist =
+		Vec3::from(joint_1.position).distance(Vec3::from(joint_2.position)) - combined_radius;
+	(1.0 - ((pinch_dist - PINCH_ACTIVACTION_DISTANCE) / (PINCH_MAX - PINCH_ACTIVACTION_DISTANCE)))
+		.clamp(0.0, 1.0)
+}
+
+fn create_trackers(session: Res<OxrSession>, mut hands: ResMut<Hands>) {
+	hands.left.tracker = session
+		.create_hand_tracker(openxr::HandEXT::LEFT)
+		.inspect_err(|err| error!("failed to create left hand tracker"))
+		.ok();
+	hands.right.tracker = session
+		.create_hand_tracker(openxr::HandEXT::RIGHT)
+		.inspect_err(|err| error!("failed to create right hand tracker"))
+		.ok();
+}
+fn destroy_trackers(mut hands: ResMut<Hands>) {
+	hands.left.tracker.take();
+	hands.right.tracker.take();
+}
+#[derive(Component)]
+struct CorrectHandMaterial;
+fn update_hand_material(
+	query: Query<
+		(Entity, &HandSide),
+		(
+			With<XrHandBoneEntities>,
+			With<MeshMaterial3d<BevyMaterial>>,
+			Without<CorrectHandMaterial>,
+		),
+	>,
+	mut cmds: Commands,
+	hands: Res<Hands>,
+) {
+	for (entity, side) in &query {
+		let handle = match side {
+			HandSide::Left => hands.left.material.clone(),
+			HandSide::Right => hands.right.material.clone(),
+		};
+		cmds.entity(entity)
+			.insert(MeshMaterial3d(handle))
+			.insert(CorrectHandMaterial);
+	}
+}
+
+fn setup(
+	connection: Res<DbusConnection>,
+	mut cmds: Commands,
+	mut materials: ResMut<Assets<BevyMaterial>>,
+) {
+	tokio::task::spawn({
+		let connection = connection.clone();
+		async move {
+			connection
+				.request_name("org.stardustxr.Hands")
+				.await
+				.unwrap();
+		}
+	});
+	cmds.insert_resource(Hands {
+		left: OxrHandInput::new(&connection, HandSide::Left, &mut materials).unwrap(),
+		right: OxrHandInput::new(&connection, HandSide::Right, &mut materials).unwrap(),
+	});
+}
+
+fn convert_joint(joint: HandJointLocation) -> Joint {
+	Joint {
+		position: joint.pose.position.to_vec3().into(),
+		rotation: joint.pose.orientation.to_quat().into(),
+		radius: joint.radius,
+		distance: 0.0,
+	}
+}
+
+#[derive(Resource)]
+struct Hands {
+	left: OxrHandInput,
+	right: OxrHandInput,
+}
+
+#[derive(Default, Deserialize, Serialize)]
+struct HandDatamap {
+	pinch_strength: f32,
+	grab_strength: f32,
+}
+
+pub struct OxrHandInput {
+	_node: OwnedNode,
+	palm_spatial: Arc<Spatial>,
+	palm_object: ObjectHandle<SpatialRef>,
+	side: HandSide,
+	input: Arc<InputMethod>,
+	capture_manager: CaptureManager,
+	datamap: HandDatamap,
+	tracked: ObjectHandle<Tracked>,
+	tracker: Option<openxr::HandTracker>,
+	captured: bool,
+	material: Handle<BevyMaterial>,
+}
+impl OxrHandInput {
+	pub fn new(
+		connection: &Connection,
+		side: HandSide,
+		materials: &mut Assets<BevyMaterial>,
+	) -> Result<Self> {
+		let (palm_spatial, palm_object) = SpatialRef::create(
+			connection,
+			&("/org/stardustxr/Hand/".to_string()
+				+ match side {
+					HandSide::Left => "left",
+					HandSide::Right => "right",
+				} + "/palm"),
+		);
+		let tracked = Tracked::new(
+			connection,
+			&("/org/stardustxr/Hand/".to_string()
+				+ match side {
+					HandSide::Left => "left",
+					HandSide::Right => "right",
+				}),
+		);
+		let node = Node::generate(&INTERNAL_CLIENT, false).add_to_scenegraph_owned()?;
+		Spatial::add_to(&node.0, None, Mat4::IDENTITY, false);
+		let hand = InputDataType::Hand(Hand {
+			right: matches!(side, HandSide::Right),
+			..Default::default()
+		});
+		let datamap = Datamap::from_typed(HandDatamap::default())?;
+		let input = InputMethod::add_to(&node.0, hand, datamap)?;
+
+		let material = materials.add(BevyMaterial {
+			base_color: Srgba::new(1.0, 1.0, 1.0, 1.0).into(),
+			alpha_mode: AlphaMode::Blend,
+			base_color_texture: Some(GRADIENT_TEXTURE_HANDLE),
+			perceptual_roughness: 1.0,
+			..default()
+		});
+		Ok(OxrHandInput {
+			_node: node,
+			palm_spatial,
+			palm_object,
+			side,
+			input,
+			tracked,
+			capture_manager: CaptureManager::default(),
+			datamap: Default::default(),
+			tracker: None,
+			material,
+			captured: false,
+		})
+	}
+	pub fn set_enabled(&self, enabled: bool) {
+		if let Some(node) = self.input.spatial.node() {
+			node.set_enabled(enabled);
+		}
+		tokio::spawn({
+			// this is suboptimal since it probably allocates a fresh string every frame
+			let handle = self.tracked.clone();
+			async move {
+				handle.set_tracked(enabled).await;
+			}
+		});
+	}
+	fn update(
+		&mut self,
+		joints: Option<&openxr::HandJointLocations>,
+		materials: &mut ResMut<Assets<BevyMaterial>>,
+	) {
+		// TODO: use the hand data source ext
+		let real_hand = true;
+		let input_node = self.input.spatial.node().unwrap();
+		let is_tracked = real_hand
+			&& joints.is_some_and(|v| {
+				v.iter().all(|v| {
+					v.location_flags.contains(
+						SpaceLocationFlags::POSITION_VALID | SpaceLocationFlags::POSITION_TRACKED,
+					) || v.location_flags.contains(
+						SpaceLocationFlags::ORIENTATION_VALID
+							| SpaceLocationFlags::ORIENTATION_TRACKED,
+					)
+				})
+			});
+		self.set_enabled(is_tracked);
+		if is_tracked {
+			// cannot ever crash, is_tracked is only true of joints is some
+			let joints = joints.unwrap();
+			let new_hand = Hand {
+				right: matches!(self.side, HandSide::Right),
+				thumb: Thumb {
+					tip: convert_joint(joints[HandBone::ThumbTip as usize]),
+					distal: convert_joint(joints[HandBone::ThumbDistal as usize]),
+					proximal: convert_joint(joints[HandBone::ThumbProximal as usize]),
+					metacarpal: convert_joint(joints[HandBone::ThumbMetacarpal as usize]),
+				},
+				index: Finger {
+					tip: convert_joint(joints[HandBone::IndexTip as usize]),
+					distal: convert_joint(joints[HandBone::IndexDistal as usize]),
+					intermediate: convert_joint(joints[HandBone::IndexIntermediate as usize]),
+					proximal: convert_joint(joints[HandBone::IndexProximal as usize]),
+					metacarpal: convert_joint(joints[HandBone::IndexMetacarpal as usize]),
+				},
+				middle: Finger {
+					tip: convert_joint(joints[HandBone::MiddleTip as usize]),
+					distal: convert_joint(joints[HandBone::MiddleDistal as usize]),
+					intermediate: convert_joint(joints[HandBone::MiddleIntermediate as usize]),
+					proximal: convert_joint(joints[HandBone::MiddleProximal as usize]),
+					metacarpal: convert_joint(joints[HandBone::MiddleMetacarpal as usize]),
+				},
+				ring: Finger {
+					tip: convert_joint(joints[HandBone::RingTip as usize]),
+					distal: convert_joint(joints[HandBone::RingDistal as usize]),
+					intermediate: convert_joint(joints[HandBone::RingIntermediate as usize]),
+					proximal: convert_joint(joints[HandBone::RingProximal as usize]),
+					metacarpal: convert_joint(joints[HandBone::RingMetacarpal as usize]),
+				},
+				little: Finger {
+					tip: convert_joint(joints[HandBone::LittleTip as usize]),
+					distal: convert_joint(joints[HandBone::LittleDistal as usize]),
+					intermediate: convert_joint(joints[HandBone::LittleIntermediate as usize]),
+					proximal: convert_joint(joints[HandBone::LittleProximal as usize]),
+					metacarpal: convert_joint(joints[HandBone::LittleMetacarpal as usize]),
+				},
+				palm: convert_joint(joints[HandBone::Palm as usize]),
+				wrist: convert_joint(joints[HandBone::Wrist as usize]),
+				elbow: None,
+			};
+			self.palm_spatial
+				.set_local_transform(Mat4::from_rotation_translation(
+					new_hand.palm.rotation.into(),
+					new_hand.palm.position.into(),
+				));
+
+			self.datamap.pinch_strength = pinch_between(&new_hand.thumb.tip, &new_hand.index.tip);
+			// this is how stereokit calculates grab
+			self.datamap.grab_strength =
+				pinch_between(&new_hand.ring.tip, &new_hand.ring.metacarpal);
+
+			*self.input.data.lock() = InputDataType::Hand(new_hand);
+			*self.input.datamap.lock() = Datamap::from_typed(&self.datamap).unwrap();
+			let captured = self.capture_manager.capture.upgrade().is_some();
+			if captured && !self.captured {
+				materials.get_mut(&self.material).unwrap().base_color =
+					Srgba::rgb(0., 1., 0.75).into();
+			} else if self.captured && !captured {
+				materials.get_mut(&self.material).unwrap().base_color =
+					Srgba::rgb(1., 1.0, 1.0).into();
+			}
+			self.captured = captured;
+		}
+
+		let distance_calculator = |space: &Arc<Spatial>, data: &InputDataType, field: &Field| {
+			let InputDataType::Hand(hand) = data else {
+				return None;
+			};
+			let thumb_tip_distance = field.distance(space, hand.thumb.tip.position.into());
+			let index_tip_distance = field.distance(space, hand.index.tip.position.into());
+			let middle_tip_distance = field.distance(space, hand.middle.tip.position.into());
+			let ring_tip_distance = field.distance(space, hand.ring.tip.position.into());
+
+			Some(
+				(thumb_tip_distance * 0.3)
+					+ (index_tip_distance * 0.4)
+					+ (middle_tip_distance * 0.15)
+					+ (ring_tip_distance * 0.15),
+			)
+		};
+
+		self.capture_manager.update_capture(&self.input);
+		self.capture_manager
+			.set_new_capture(&self.input, distance_calculator);
+		self.capture_manager.apply_capture(&self.input);
+
+		if self.capture_manager.capture.upgrade().is_some() {
+			return;
+		}
+
+		let sorted_handlers = get_sorted_handlers(&self.input, distance_calculator);
+		self.input
+			.set_handler_order(sorted_handlers.iter().map(|(handler, _)| handler));
+	}
+}

src/objects/input/sk_controller.rs

@@ -1,47 +0,0 @@
-use crate::nodes::{
-	input::{tip::Tip, InputMethod, InputType},
-	spatial::Spatial,
-};
-use glam::Mat4;
-use stardust_xr::{schemas::flat::Datamap, values::Transform};
-use std::sync::{Arc, Weak};
-use stereokit::{
-	input::{ButtonState, Handed},
-	StereoKit,
-};
-
-pub struct SkController {
-	tip: Arc<InputMethod>,
-	handed: Handed,
-}
-impl SkController {
-	pub fn new(handed: Handed) -> Self {
-		SkController {
-			tip: InputMethod::new(
-				Spatial::new(Weak::new(), None, Mat4::IDENTITY),
-				InputType::Tip(Tip::default()),
-			),
-			handed,
-		}
-	}
-	pub fn update(&mut self, sk: &StereoKit) {
-		let controller = sk.input_controller(self.handed);
-		*self.tip.enabled.lock() = controller.tracked.contains(ButtonState::Active);
-		if *self.tip.enabled.lock() {
-			self.tip.spatial.set_local_transform_components(
-				None,
-				Transform {
-					position: Some(controller.pose.position),
-					rotation: Some(controller.pose.orientation),
-					scale: None,
-				},
-			);
-		}
-		let mut fbb = flexbuffers::Builder::default();
-		let mut map = fbb.start_map();
-		map.push("select", controller.trigger);
-		map.push("grab", controller.grip);
-		map.end_map();
-		*self.tip.datamap.lock() = Datamap::new(fbb.take_buffer()).ok();
-	}
-}

src/objects/input/sk_hand.rs

@@ -1,85 +0,0 @@
-use crate::nodes::{
-	input::{hand::Hand, InputMethod, InputType},
-	spatial::Spatial,
-};
-use glam::Mat4;
-use stardust_xr::schemas::flat::{Datamap, Hand as FlatHand, Joint};
-use std::sync::{Arc, Weak};
-use stereokit::{
-	input::{ButtonState, Handed, Joint as SkJoint},
-	StereoKit,
-};
-
-fn convert_joint(joint: SkJoint) -> Joint {
-	Joint {
-		position: joint.position,
-		rotation: joint.orientation,
-		radius: joint.radius,
-	}
-}
-
-pub struct SkHand {
-	hand: Arc<InputMethod>,
-	handed: Handed,
-}
-impl SkHand {
-	pub fn new(handed: Handed) -> Self {
-		SkHand {
-			hand: InputMethod::new(
-				Spatial::new(Weak::new(), None, Mat4::IDENTITY),
-				InputType::Hand(Box::new(Hand {
-					base: FlatHand {
-						right: handed == Handed::Right,
-						..Default::default()
-					},
-				})),
-			),
-			handed,
-		}
-	}
-	pub fn update(&mut self, sk: &StereoKit) {
-		let sk_hand = sk.input_hand(self.handed);
-		if let InputType::Hand(hand) = &mut *self.hand.specialization.lock() {
-			let controller = sk.input_controller(self.handed);
-			*self.hand.enabled.lock() = controller.tracked.contains(ButtonState::Inactive)
-				&& sk_hand.tracked_state.contains(ButtonState::Active);
-			if *self.hand.enabled.lock() {
-				hand.base.thumb.tip = convert_joint(sk_hand.fingers[0][4]);
-				hand.base.thumb.distal = convert_joint(sk_hand.fingers[0][3]);
-				hand.base.thumb.proximal = convert_joint(sk_hand.fingers[0][2]);
-				hand.base.thumb.metacarpal = convert_joint(sk_hand.fingers[0][1]);
-
-				for (finger, sk_finger) in [
-					(&mut hand.base.index, sk_hand.fingers[1]),
-					(&mut hand.base.middle, sk_hand.fingers[2]),
-					(&mut hand.base.ring, sk_hand.fingers[3]),
-					(&mut hand.base.little, sk_hand.fingers[4]),
-				] {
-					finger.tip = convert_joint(sk_finger[4]);
-					finger.distal = convert_joint(sk_finger[3]);
-					finger.intermediate = convert_joint(sk_finger[2]);
-					finger.proximal = convert_joint(sk_finger[1]);
-					finger.metacarpal = convert_joint(sk_finger[0]);
-				}
-
-				hand.base.palm.position = sk_hand.palm.position;
-				hand.base.palm.rotation = sk_hand.palm.orientation;
-				hand.base.palm.radius =
-					(sk_hand.fingers[2][0].radius + sk_hand.fingers[2][1].radius) * 0.5;
-
-				hand.base.wrist.position = sk_hand.wrist.position;
-				hand.base.wrist.rotation = sk_hand.wrist.orientation;
-				hand.base.wrist.radius =
-					(sk_hand.fingers[0][0].radius + sk_hand.fingers[4][0].radius) * 0.5;
-
-				hand.base.elbow = None;
-			}
-		}
-		let mut fbb = flexbuffers::Builder::default();
-		let mut map = fbb.start_map();
-		map.push("grabStrength", sk_hand.grip_activation);
-		map.push("pinchStrength", sk_hand.pinch_activation);
-		map.end_map();
-		*self.hand.datamap.lock() = Datamap::new(fbb.take_buffer()).ok();
-	}
-}

src/objects/mod.rs

@@ -1 +1,171 @@
+#![allow(unused)]
+
+use crate::{
+	core::client::INTERNAL_CLIENT,
+	nodes::{
+		Node, OwnedNode,
+		fields::{EXPORTED_FIELDS, Field, Shape},
+		spatial::{EXPORTED_SPATIALS, Spatial},
+	},
+};
+use glam::{Mat4, vec3};
+use input::{
+	eye_pointer::EyePointer, mouse_pointer::MousePointer, oxr_controller::OxrControllerInput,
+	oxr_hand::OxrHandInput,
+};
+use parking_lot::RwLock;
+use play_space::PlaySpaceBounds;
+use stardust_xr::schemas::dbus::object_registry::ObjectRegistry;
+use std::{
+	marker::PhantomData,
+	sync::{Arc, atomic::Ordering},
+};
+use zbus::{Connection, interface, object_server::Interface, zvariant::OwnedObjectPath};
+
+pub mod hmd;
 pub mod input;
+pub mod play_space;
+
+pub struct ObjectHandle<I: Interface>(Connection, OwnedObjectPath, PhantomData<I>);
+
+impl<I: Interface> Clone for ObjectHandle<I> {
+	fn clone(&self) -> Self {
+		Self(self.0.clone(), self.1.clone(), PhantomData)
+	}
+}
+impl<I: Interface> Drop for ObjectHandle<I> {
+	fn drop(&mut self) {
+		let connection = self.0.clone();
+		let object_path = self.1.clone();
+		tokio::task::spawn(async move {
+			connection.object_server().remove::<I, _>(object_path);
+		});
+	}
+}
+
+pub struct SpatialRef(u64, OwnedNode);
+impl SpatialRef {
+	pub fn create(connection: &Connection, path: &str) -> (Arc<Spatial>, ObjectHandle<SpatialRef>) {
+		let node = OwnedNode(Arc::new(Node::generate(&INTERNAL_CLIENT, false)));
+		let spatial = Spatial::add_to(&node.0, None, Mat4::IDENTITY, false);
+		let uid: u64 = rand::random();
+		EXPORTED_SPATIALS
+			.lock()
+			.insert(uid, Arc::downgrade(&node.0));
+
+		tokio::task::spawn({
+			let connection = connection.clone();
+			let path = path.to_string();
+			async move {
+				connection
+					.object_server()
+					.at(path, Self(uid, node))
+					.await
+					.unwrap();
+			}
+		});
+		(
+			spatial,
+			ObjectHandle(
+				connection.clone(),
+				OwnedObjectPath::try_from(path.to_string()).unwrap(),
+				PhantomData,
+			),
+		)
+	}
+}
+#[interface(name = "org.stardustxr.SpatialRef")]
+impl SpatialRef {
+	#[zbus(property)]
+	fn uid(&self) -> u64 {
+		self.0
+	}
+}
+
+pub struct Tracked(bool);
+impl Tracked {
+	pub fn new(connection: &Connection, path: &str) -> ObjectHandle<Tracked> {
+		tokio::task::spawn({
+			let connection = connection.clone();
+			let path = path.to_string();
+			async move {
+				connection
+					.object_server()
+					.at(path, Self(false))
+					.await
+					.unwrap();
+			}
+		});
+		ObjectHandle(
+			connection.clone(),
+			OwnedObjectPath::try_from(path.to_string()).unwrap(),
+			PhantomData,
+		)
+	}
+}
+impl ObjectHandle<Tracked> {
+	pub async fn set_tracked(&self, is_tracked: bool) -> zbus::Result<()> {
+		let tracked_ref = self
+			.0
+			.object_server()
+			.interface::<_, Tracked>(self.1.as_ref())
+			.await?;
+		let mut tracked = tracked_ref.get_mut().await;
+		if tracked.0 != is_tracked {
+			tracked.0 = is_tracked;
+			tracked
+				.is_tracked_changed(tracked_ref.signal_emitter())
+				.await;
+		}
+		Ok(())
+	}
+}
+#[interface(name = "org.stardustxr.Tracked")]
+impl Tracked {
+	#[zbus(property)]
+	fn is_tracked(&self) -> bool {
+		self.0
+	}
+}
+
+pub struct FieldRef(u64, OwnedNode);
+impl FieldRef {
+	pub fn create(
+		connection: &Connection,
+		path: &str,
+		shape: Shape,
+	) -> (Arc<Field>, ObjectHandle<FieldRef>) {
+		let node = OwnedNode(Arc::new(Node::generate(&INTERNAL_CLIENT, false)));
+		Spatial::add_to(&node.0, None, Mat4::IDENTITY, false);
+		let field = Field::add_to(&node.0, shape).unwrap();
+		let uid: u64 = rand::random();
+		EXPORTED_FIELDS.lock().insert(uid, Arc::downgrade(&node.0));
+
+		tokio::task::spawn({
+			let connection = connection.clone();
+			let path = path.to_string();
+			async move {
+				connection
+					.object_server()
+					.at(path, Self(uid, node))
+					.await
+					.unwrap();
+			}
+		});
+		(
+			field,
+			ObjectHandle(
+				connection.clone(),
+				OwnedObjectPath::try_from(path.to_string()).unwrap(),
+				PhantomData,
+			),
+		)
+	}
+}
+#[interface(name = "org.stardustxr.FieldRef")]
+impl FieldRef {
+	#[zbus(property)]
+	fn uid(&self) -> u64 {
+		self.0
+	}
+}

src/objects/play_space.rs

@@ -0,0 +1,162 @@
+use std::sync::Arc;
+
+use bevy::prelude::*;
+use bevy_mod_openxr::{
+	helper_traits::{ToQuat, ToVec3},
+	resources::OxrFrameState,
+	session::OxrSession,
+};
+use bevy_mod_xr::{
+	session::{XrPreDestroySession, XrSessionCreated},
+	spaces::{XrPrimaryReferenceSpace, XrReferenceSpace, XrSpace},
+};
+use openxr::SpaceLocationFlags;
+use parking_lot::RwLock;
+use zbus::{Connection, ObjectServer, interface};
+
+use crate::{DbusConnection, PreFrameWait, nodes::spatial::Spatial};
+
+use super::{ObjectHandle, SpatialRef, Tracked};
+
+pub struct PlaySpacePlugin;
+impl Plugin for PlaySpacePlugin {
+	fn build(&self, app: &mut App) {
+		app.add_systems(XrPreDestroySession, destroy_stage_space);
+		app.add_systems(XrSessionCreated, create_stage_space);
+		app.add_systems(PreFrameWait, update);
+		app.add_systems(Startup, setup);
+	}
+}
+
+fn setup(connection: Res<DbusConnection>, mut cmds: Commands) {
+	let (spatial, spatial_handle) = SpatialRef::create(&connection, "/org/stardustxr/PlaySpace");
+	// the OpenXR session might not exist quite yet
+	let tracked = Tracked::new(&connection, "/org/stardustxr/PlaySpace");
+	let dbus_connection = connection.clone();
+	let play_space_data = Arc::new(RwLock::default());
+	tokio::task::spawn({
+		let data = play_space_data.clone();
+		async move {
+			PlaySpaceBounds::create(&dbus_connection, data).await;
+			dbus_connection
+				.request_name("org.stardustxr.PlaySpace")
+				.await
+				.unwrap();
+		}
+	});
+	cmds.insert_resource(PlaySpace {
+		spatial,
+		_spatial_handle: spatial_handle,
+		tracked_handle: tracked,
+		bounds: play_space_data,
+	});
+}
+
+#[derive(Resource)]
+struct StageSpace(XrSpace);
+fn create_stage_space(session: Res<OxrSession>, mut cmds: Commands) {
+	let space = session
+		.create_reference_space(openxr::ReferenceSpaceType::STAGE, Transform::IDENTITY)
+		.inspect_err(|err| error!("failed to create Stage XrSpace"))
+		.ok();
+	if let Some(space) = space {
+		cmds.insert_resource(StageSpace(space.0));
+	}
+}
+fn destroy_stage_space(session: Res<OxrSession>, mut cmds: Commands, stage: Res<StageSpace>) {
+	session.destroy_space(stage.0);
+	cmds.remove_resource::<StageSpace>();
+}
+
+/// TODO: impl this
+fn update(
+	session: Option<Res<OxrSession>>,
+	stage: Option<Res<StageSpace>>,
+	ref_space: Option<Res<XrPrimaryReferenceSpace>>,
+	play_space: Res<PlaySpace>,
+	state: Option<Res<OxrFrameState>>,
+) {
+	let (Some(session), Some(stage), Some(ref_space), Some(state)) =
+		(session, stage, ref_space, state)
+	else {
+		play_space.bounds.write().drain(..);
+		tokio::task::spawn({
+			let handle = play_space.tracked_handle.clone();
+			async move {
+				handle.set_tracked(false).await;
+			}
+		});
+
+		play_space
+			.spatial
+			.set_local_transform(Mat4::from_translation(vec3(0.0, -1.65, 0.0)));
+		return;
+	};
+	// this won't be correct with pipelined rendering
+	let location = session
+		.locate_space(&stage.0, &ref_space, state.predicted_display_time)
+		.inspect_err(|err| error!("Error while Locating OpenXR Stage Space {err}"));
+	if let Ok(location) = location {
+		let is_tracked = location.location_flags.contains(
+			SpaceLocationFlags::POSITION_VALID
+				| SpaceLocationFlags::POSITION_TRACKED
+				| SpaceLocationFlags::ORIENTATION_VALID
+				| SpaceLocationFlags::ORIENTATION_TRACKED,
+		);
+		tokio::task::spawn({
+			let handle = play_space.tracked_handle.clone();
+			async move {
+				handle.set_tracked(is_tracked).await;
+			}
+		});
+		if is_tracked {
+			play_space
+				.spatial
+				.set_local_transform(Mat4::from_rotation_translation(
+					location.pose.orientation.to_quat(),
+					location.pose.position.to_vec3(),
+				));
+		}
+	}
+	// session.reference_space_bounds_rect(openxr::ReferenceSpaceType::STAGE);
+
+	// if (World::has_bounds()
+	// 	&& World::get_bounds_size().x != 0.0
+	// 	&& World::get_bounds_size().y != 0.0)
+	// {
+	// 	let bounds = World::get_bounds_size();
+	// 	vec![
+	// 		((bounds.x).into(), (bounds.y).into()),
+	// 		((bounds.x).into(), (-bounds.y).into()),
+	// 		((-bounds.x).into(), (-bounds.y).into()),
+	// 		((-bounds.x).into(), (bounds.y).into()),
+	// 	]
+	// } else {
+	// 	vec![]
+	// }
+}
+
+#[derive(Resource)]
+pub struct PlaySpace {
+	spatial: Arc<Spatial>,
+	_spatial_handle: ObjectHandle<SpatialRef>,
+	tracked_handle: ObjectHandle<Tracked>,
+	bounds: Arc<RwLock<Vec<(f64, f64)>>>,
+}
+pub struct PlaySpaceBounds(Arc<RwLock<Vec<(f64, f64)>>>);
+impl PlaySpaceBounds {
+	pub async fn create(connection: &Connection, data: Arc<RwLock<Vec<(f64, f64)>>>) {
+		connection
+			.object_server()
+			.at("/org/stardustxr/PlaySpace", Self(data))
+			.await
+			.unwrap();
+	}
+}
+#[interface(name = "org.stardustxr.PlaySpace")]
+impl PlaySpaceBounds {
+	#[zbus(property)]
+	fn bounds(&self) -> Vec<(f64, f64)> {
+		self.0.read().clone()
+	}
+}

src/session.rs

@@ -0,0 +1,110 @@
+use crate::core::client::CLIENTS;
+use crate::core::client_state::ClientStateParsed;
+#[cfg(feature = "wayland")]
+use crate::wayland::WAYLAND_DISPLAY;
+use crate::{CliArgs, STARDUST_INSTANCE};
+use directories::ProjectDirs;
+use rustc_hash::FxHashMap;
+use std::os::unix::fs::PermissionsExt;
+use std::path::Path;
+use std::process::{Child, Command, Stdio};
+use std::time::Duration;
+use tokio::task::LocalSet;
+use tracing::info;
+
+pub async fn save_session(project_dirs: &ProjectDirs) {
+	let session_id = nanoid::nanoid!();
+	let state_dir = project_dirs.state_dir().unwrap();
+	let session_dir = state_dir.join(&session_id);
+	std::fs::create_dir_all(&session_dir).unwrap();
+	let _ = std::fs::remove_dir_all(state_dir.join("latest"));
+	std::os::unix::fs::symlink(&session_dir, state_dir.join("latest")).unwrap();
+
+	let local_set = LocalSet::new();
+	for client in CLIENTS.get_vec() {
+		let session_dir = session_dir.clone();
+		local_set.spawn_local(async move {
+			tokio::select! {
+				biased;
+				s = client.save_state() => {if let Some(s) = s { s.to_file(&session_dir) }},
+				_ = tokio::time::sleep(Duration::from_millis(100)) => (),
+			}
+		});
+	}
+	local_set.await;
+	info!("Session ID for restore is {session_id}");
+}
+
+pub fn launch_start(cli_args: &CliArgs, project_dirs: &ProjectDirs) -> Vec<Child> {
+	match (&cli_args.restore, &cli_args.startup_script) {
+		(Some(session_id), _) => restore_session(
+			&project_dirs.state_dir().unwrap().join(session_id),
+			cli_args.debug_launched_clients,
+		),
+		(None, Some(startup_script)) => run_script(
+			&startup_script.clone().canonicalize().unwrap_or_default(),
+			cli_args.debug_launched_clients,
+		),
+		(None, None) => run_script(
+			&project_dirs.config_dir().join("startup"),
+			cli_args.debug_launched_clients,
+		),
+	}
+}
+
+pub fn restore_session(session_dir: &Path, debug_launched_clients: bool) -> Vec<Child> {
+	let Ok(clients) = session_dir.read_dir() else {
+		return Vec::new();
+	};
+	clients
+		.filter_map(Result::ok)
+		.filter_map(|c| ClientStateParsed::from_file(&c.path()))
+		.filter_map(ClientStateParsed::launch_command)
+		.filter_map(|c| run_client(c, debug_launched_clients))
+		.collect()
+}
+
+pub fn run_script(script_path: &Path, debug_launched_clients: bool) -> Vec<Child> {
+	let _ = std::fs::set_permissions(script_path, std::fs::Permissions::from_mode(0o755));
+	let startup_command = Command::new(script_path);
+	run_client(startup_command, debug_launched_clients)
+		.map(|c| vec![c])
+		.unwrap_or_default()
+}
+
+pub fn run_client(mut command: Command, debug_launched_clients: bool) -> Option<Child> {
+	command.stdin(Stdio::null());
+	if !debug_launched_clients {
+		command.stdout(Stdio::null());
+		command.stderr(Stdio::null());
+	}
+	for (var, value) in connection_env() {
+		command.env(var, value);
+	}
+	let child = command.spawn().ok()?;
+	Some(child)
+}
+
+pub fn connection_env() -> FxHashMap<String, String> {
+	let mut env: FxHashMap<String, String> = FxHashMap::default();
+	env.insert(
+		stringify!(STARDUST_INSTANCE).to_string(),
+		STARDUST_INSTANCE.get().unwrap().clone(),
+	);
+
+	if let Some(flat_wayland_display) = std::env::var_os("WAYLAND_DISPLAY") {
+		env.insert(
+			"FLAT_WAYLAND_DISPLAY".to_string(),
+			flat_wayland_display.to_string_lossy().into_owned(),
+		);
+	}
+	#[cfg(feature = "wayland")]
+	{
+		env.insert(
+			stringify!(WAYLAND_DISPLAY).to_string(),
+			WAYLAND_DISPLAY.get().unwrap().to_string_lossy().to_string(),
+		);
+		env.insert("XDG_SESSION_TYPE".to_string(), "wayland".to_string());
+	}
+	env
+}

src/tracking_offset.rs

@@ -0,0 +1,82 @@
+use bevy::{
+	app::{Plugin, Update},
+	ecs::{
+		resource::Resource,
+		schedule::{Condition, IntoScheduleConfigs, common_conditions::resource_exists},
+		system::{Commands, Res, ResMut},
+	},
+	transform::components::Transform,
+};
+use bevy_mod_openxr::{
+	helper_traits::ToTransform as _,
+	poll_events::{OxrEventHandlerExt, OxrEventIn},
+	resources::OxrFrameState,
+	session::OxrSession,
+};
+use bevy_mod_xr::{session::XrSessionCreated, spaces::XrPrimaryReferenceSpace};
+use glam::{Quat, Vec3};
+use openxr::ReferenceSpaceType;
+
+pub struct TrackingOffsetPlugin;
+
+impl Plugin for TrackingOffsetPlugin {
+	fn build(&self, app: &mut bevy::app::App) {
+		app.add_oxr_event_handler(reset_offset);
+		app.add_systems(XrSessionCreated, |mut cmds: Commands| {
+			cmds.insert_resource(OffsetTag);
+		});
+		app.add_systems(
+			Update,
+			offset.run_if(resource_exists::<OffsetTag>.and(resource_exists::<OxrFrameState>)),
+		);
+	}
+}
+
+#[derive(Resource)]
+struct OffsetTag;
+
+fn reset_offset(
+	oxr_event: OxrEventIn,
+	mut ref_space: ResMut<XrPrimaryReferenceSpace>,
+	session: Res<OxrSession>,
+) {
+	if let openxr::Event::ReferenceSpaceChangePending(v) = *oxr_event
+		&& v.reference_space_type() == ReferenceSpaceType::LOCAL
+	{
+		let space = session
+			.create_reference_space(ReferenceSpaceType::LOCAL, Transform::IDENTITY)
+			.unwrap();
+		session.destroy_space(ref_space.0.0).unwrap();
+		ref_space.0 = space;
+	}
+}
+
+fn offset(
+	session: Res<OxrSession>,
+	state: Res<OxrFrameState>,
+	mut primary_ref_space: ResMut<XrPrimaryReferenceSpace>,
+	mut cmds: Commands,
+) {
+	cmds.remove_resource::<OffsetTag>();
+	let local = session
+		.create_reference_space(ReferenceSpaceType::LOCAL, Transform::IDENTITY)
+		.unwrap();
+	let view = session
+		.create_reference_space(ReferenceSpaceType::VIEW, Transform::IDENTITY)
+		.unwrap();
+	let view_pose = session
+		.locate_space(&view, &local, state.predicted_display_time)
+		.unwrap()
+		.pose
+		.to_transform();
+	let offset = view_pose.with_rotation(Quat::from_axis_angle(
+		Vec3::Y,
+		view_pose.rotation.to_euler(glam::EulerRot::XYZ).1,
+	));
+	let offset = Transform::from_matrix(offset.compute_matrix());
+	let local_offset = session
+		.create_reference_space(ReferenceSpaceType::LOCAL, offset)
+		.unwrap();
+	session.destroy_space(primary_ref_space.0.0).unwrap();
+	primary_ref_space.0 = local_offset;
+}

src/wayland/compositor.rs

@@ -1,27 +0,0 @@
-use super::{state::WaylandState, surface::CoreSurface};
-use smithay::{
-	delegate_compositor,
-	reexports::wayland_server::protocol::wl_surface::WlSurface,
-	wayland::compositor::{self, CompositorHandler, CompositorState},
-};
-
-impl CompositorHandler for WaylandState {
-	fn compositor_state(&mut self) -> &mut CompositorState {
-		&mut self.compositor_state
-	}
-
-	fn commit(&mut self, surface: &WlSurface) {
-		compositor::with_states(surface, |data| {
-			data.data_map.insert_if_missing_threadsafe(|| {
-				CoreSurface::new(
-					&self.weak_ref.upgrade().unwrap(),
-					&self.display,
-					self.display_handle.clone(),
-					surface,
-				)
-			})
-		});
-	}
-}
-
-delegate_compositor!(WaylandState);

src/wayland/core/buffer.rs

@@ -0,0 +1,94 @@
+use crate::wayland::Message;
+use crate::wayland::dmabuf::buffer_backing::DmabufBacking;
+use crate::wayland::{MessageSink, core::shm_buffer_backing::ShmBufferBacking, util::ClientExt};
+use bevy::{
+	asset::{Assets, Handle},
+	image::Image,
+};
+use bevy_dmabuf::import::ImportedDmatexs;
+use mint::Vector2;
+use std::sync::Arc;
+pub use waynest::server::protocol::core::wayland::wl_buffer::*;
+use waynest::{
+	server::{Client, Dispatcher, Result},
+	wire::ObjectId,
+};
+
+#[derive(Debug)]
+pub struct BufferUsage {
+	pub buffer: Arc<Buffer>,
+	message_sink: MessageSink,
+}
+impl BufferUsage {
+	pub fn new(client: &Client, buffer: &Arc<Buffer>) -> Arc<Self> {
+		Arc::new(Self {
+			buffer: buffer.clone(),
+			message_sink: client.message_sink(),
+		})
+	}
+}
+impl Drop for BufferUsage {
+	fn drop(&mut self) {
+		let _ = self
+			.message_sink
+			.send(Message::ReleaseBuffer(self.buffer.clone()));
+	}
+}
+
+#[derive(Debug)]
+pub enum BufferBacking {
+	Shm(ShmBufferBacking),
+	Dmabuf(DmabufBacking),
+}
+
+#[derive(Debug, Dispatcher)]
+pub struct Buffer {
+	pub id: ObjectId,
+	backing: BufferBacking,
+}
+
+impl Buffer {
+	#[tracing::instrument(level = "debug", skip_all)]
+	pub fn new(client: &mut Client, id: ObjectId, backing: BufferBacking) -> Arc<Self> {
+		client.insert(id, Self { id, backing })
+	}
+
+	/// Returns the tex if it was updated
+	#[tracing::instrument(level = "debug", skip_all)]
+	pub fn update_tex(
+		&self,
+		dmatexes: &ImportedDmatexs,
+		images: &mut Assets<Image>,
+	) -> Option<Handle<Image>> {
+		tracing::debug!("Updating texture for buffer {:?}", self.id);
+		match &self.backing {
+			BufferBacking::Shm(backing) => backing.update_tex(images),
+			BufferBacking::Dmabuf(backing) => backing.update_tex(dmatexes, images),
+		}
+	}
+
+	pub fn is_transparent(&self) -> bool {
+		match &self.backing {
+			BufferBacking::Shm(backing) => backing.is_transparent(),
+			BufferBacking::Dmabuf(backing) => backing.is_transparent(),
+		}
+	}
+
+	pub fn size(&self) -> Vector2<usize> {
+		match &self.backing {
+			BufferBacking::Shm(backing) => backing.size(),
+			BufferBacking::Dmabuf(backing) => backing.size(),
+		}
+	}
+	pub fn uses_buffer_usage(&self) -> bool {
+		matches!(self.backing, BufferBacking::Dmabuf(_))
+	}
+}
+
+impl WlBuffer for Buffer {
+	/// https://wayland.app/protocols/wayland#wl_buffer:request:destroy
+	async fn destroy(&self, _client: &mut Client, _sender_id: ObjectId) -> Result<()> {
+		tracing::info!("Destroying buffer {:?}", self.id);
+		Ok(())
+	}
+}

src/wayland/core/callback.rs

@@ -0,0 +1,10 @@
+pub use waynest::server::protocol::core::wayland::wl_callback::*;
+use waynest::{
+	server::{Dispatcher, Result},
+	wire::ObjectId,
+};
+
+#[derive(Debug, Dispatcher, Clone)]
+pub struct Callback(pub ObjectId);
+/// https://wayland.app/protocols/wayland#wl_callback
+impl WlCallback for Callback {}

src/wayland/core/compositor.rs

@@ -0,0 +1,76 @@
+use super::surface::WL_SURFACE_REGISTRY;
+use crate::wayland::{core::surface::Surface, util::ClientExt};
+pub use waynest::server::protocol::core::wayland::wl_compositor::*;
+use waynest::{
+	server::{
+		Client, Dispatcher, Result,
+		protocol::core::wayland::{wl_region::WlRegion, wl_surface::WlSurface},
+	},
+	wire::ObjectId,
+};
+
+#[derive(Debug, Dispatcher, Default)]
+pub struct Compositor;
+impl WlCompositor for Compositor {
+	/// https://wayland.app/protocols/wayland#wl_compositor:request:create_surface
+	async fn create_surface(
+		&self,
+		client: &mut Client,
+		_sender_id: ObjectId,
+		id: ObjectId,
+	) -> Result<()> {
+		let surface = client.insert(id, Surface::new(client, id));
+		if let Some(output) = client.display().output.get() {
+			surface.enter(client, id, output.id).await?;
+		}
+		WL_SURFACE_REGISTRY.add_raw(&surface);
+
+		Ok(())
+	}
+
+	/// https://wayland.app/protocols/wayland#wl_compositor:request:create_region
+	async fn create_region(
+		&self,
+		client: &mut Client,
+		_sender_id: ObjectId,
+		id: ObjectId,
+	) -> Result<()> {
+		client.insert(id, Region::default());
+		Ok(())
+	}
+}
+
+#[derive(Debug, Dispatcher, Default)]
+pub struct Region {}
+impl WlRegion for Region {
+	/// https://wayland.app/protocols/wayland#wl_region:request:add
+	async fn add(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_x: i32,
+		_y: i32,
+		_width: i32,
+		_height: i32,
+	) -> Result<()> {
+		Ok(())
+	}
+
+	/// https://wayland.app/protocols/wayland#wl_region:request:subtract
+	async fn subtract(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_x: i32,
+		_y: i32,
+		_width: i32,
+		_height: i32,
+	) -> Result<()> {
+		Ok(())
+	}
+
+	/// https://wayland.app/protocols/wayland#wl_region:request:destroy
+	async fn destroy(&self, _client: &mut Client, _sender_id: ObjectId) -> Result<()> {
+		Ok(())
+	}
+}

src/wayland/core/data_device.rs

@@ -0,0 +1,146 @@
+use std::os::fd::OwnedFd;
+use waynest::{
+	server::{
+		Client, Dispatcher, Result,
+		protocol::core::wayland::{
+			wl_data_device::*, wl_data_device_manager::*, wl_data_offer::WlDataOffer,
+			wl_data_source::*,
+		},
+	},
+	wire::ObjectId,
+};
+
+// TODO: actually implement this
+
+#[derive(Debug, Dispatcher)]
+pub struct DataDeviceManager;
+impl WlDataDeviceManager for DataDeviceManager {
+	async fn create_data_source(
+		&self,
+		client: &mut Client,
+		_sender_id: ObjectId,
+		id: ObjectId,
+	) -> Result<()> {
+		client.insert(id, DataSource);
+		Ok(())
+	}
+
+	async fn get_data_device(
+		&self,
+		client: &mut Client,
+		_sender_id: ObjectId,
+		id: ObjectId,
+		_seat: ObjectId,
+	) -> Result<()> {
+		client.insert(id, DataDevice);
+		Ok(())
+	}
+}
+
+#[derive(Debug, Dispatcher)]
+pub struct DataSource;
+impl WlDataSource for DataSource {
+	async fn send(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_mime_type: String,
+		_fd: OwnedFd,
+	) -> Result<()> {
+		Ok(())
+	}
+
+	async fn offer(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_mime_type: String,
+	) -> Result<()> {
+		Ok(())
+	}
+
+	async fn destroy(&self, _client: &mut Client, _sender_id: ObjectId) -> Result<()> {
+		Ok(())
+	}
+
+	async fn set_actions(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_dnd_actions: DndAction,
+	) -> Result<()> {
+		Ok(())
+	}
+}
+
+#[derive(Debug, Dispatcher)]
+pub struct DataDevice;
+impl WlDataDevice for DataDevice {
+	async fn start_drag(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_source: Option<ObjectId>,
+		_origin: ObjectId,
+		_icon: Option<ObjectId>,
+		_serial: u32,
+	) -> Result<()> {
+		Ok(())
+	}
+
+	async fn set_selection(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_source: Option<ObjectId>,
+		_serial: u32,
+	) -> Result<()> {
+		Ok(())
+	}
+
+	async fn release(&self, _client: &mut Client, _sender_id: ObjectId) -> Result<()> {
+		Ok(())
+	}
+}
+
+#[derive(Debug, Dispatcher)]
+pub struct DataOffer;
+impl WlDataOffer for DataOffer {
+	async fn accept(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_serial: u32,
+		_mime_type: Option<String>,
+	) -> Result<()> {
+		Ok(())
+	}
+
+	async fn receive(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_mime_type: String,
+		_fd: OwnedFd,
+	) -> Result<()> {
+		Ok(())
+	}
+
+	async fn destroy(&self, _client: &mut Client, _sender_id: ObjectId) -> Result<()> {
+		Ok(())
+	}
+
+	async fn finish(&self, _client: &mut Client, _sender_id: ObjectId) -> Result<()> {
+		Ok(())
+	}
+
+	async fn set_actions(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_dnd_actions: DndAction,
+		_preferred_action: DndAction,
+	) -> Result<()> {
+		Ok(())
+	}
+}

src/wayland/core/keyboard.rs

@@ -0,0 +1,254 @@
+use crate::{
+	nodes::items::panel::KEYMAPS,
+	wayland::{core::surface::Surface, util::ClientExt},
+};
+use dashmap::{DashMap, DashSet};
+use memfd::MemfdOptions;
+use slotmap::{DefaultKey, KeyData};
+use std::{
+	collections::HashSet,
+	io::Write,
+	os::{
+		fd::IntoRawFd,
+		unix::io::{FromRawFd, OwnedFd},
+	},
+	sync::{Arc, Weak},
+};
+use tokio::sync::Mutex;
+pub use waynest::server::protocol::core::wayland::wl_keyboard::*;
+use waynest::{
+	server::{Client, Dispatcher, Result},
+	wire::ObjectId,
+};
+
+#[derive(Default)]
+struct ModifierState {
+	pressed_keys: HashSet<u32>,
+	mods_depressed: u32,
+	mods_latched: u32,
+	mods_locked: u32,
+	group: u32,
+}
+
+impl ModifierState {
+	fn update_key(&mut self, key: u32, pressed: bool) -> bool {
+		let changed = if pressed {
+			self.pressed_keys.insert(key)
+		} else {
+			self.pressed_keys.remove(&key)
+		};
+
+		if changed {
+			self.update_modifiers();
+		}
+		changed
+	}
+
+	fn update_modifiers(&mut self) {
+		let mut mods = 0;
+
+		// Update modifier state based on currently pressed keys
+		for key in &self.pressed_keys {
+			match *key {
+				input_event_codes::KEY_LEFTSHIFT!() | input_event_codes::KEY_RIGHTSHIFT!() => {
+					mods |= 1
+				}
+				input_event_codes::KEY_LEFTCTRL!() | input_event_codes::KEY_RIGHTCTRL!() => {
+					mods |= 4
+				}
+				input_event_codes::KEY_LEFTALT!() | input_event_codes::KEY_RIGHTALT!() => mods |= 8,
+				input_event_codes::KEY_LEFTMETA!() | input_event_codes::KEY_RIGHTMETA!() => {
+					mods |= 64
+				}
+				input_event_codes::KEY_CAPSLOCK!() => self.mods_locked ^= 1,
+				_ => {}
+			}
+		}
+
+		self.mods_depressed = mods;
+	}
+}
+
+#[derive(Dispatcher)]
+pub struct Keyboard {
+	pub id: ObjectId,
+	focused_surface: Mutex<Weak<Surface>>,
+	modifier_state: Mutex<ModifierState>,
+	pressed_keys: DashMap<ObjectId, DashSet<u32>>,
+	current_keymap_id: Mutex<u64>,
+}
+
+impl Keyboard {
+	pub fn new(id: ObjectId) -> Self {
+		Self {
+			id,
+			focused_surface: Mutex::new(Weak::new()),
+			modifier_state: Mutex::new(ModifierState::default()),
+			pressed_keys: DashMap::default(),
+			current_keymap_id: Mutex::new(0),
+		}
+	}
+
+	async fn send_keymap(&self, client: &mut Client, keymap: &[u8]) -> Result<()> {
+		let mut file = MemfdOptions::default()
+			.create("stardust-keymap")
+			.map_err(|e| waynest::server::Error::Custom(e.to_string()))?
+			.into_file();
+		file.set_len(keymap.len() as u64)?;
+		file.write_all(keymap)?;
+		file.flush()?;
+
+		let fd = unsafe { OwnedFd::from_raw_fd(file.into_raw_fd()) };
+
+		// Send keymap to client
+		self.keymap(
+			client,
+			self.id,
+			KeymapFormat::XkbV1,
+			fd,
+			keymap.len() as u32,
+		)
+		.await?;
+
+		Ok(())
+	}
+
+	/// has to be the wayland key, so -8 or whatever
+	pub async fn handle_keyboard_key(
+		&self,
+		client: &mut Client,
+		surface: Arc<Surface>,
+		keymap_id: u64,
+		key: u32,
+		pressed: bool,
+	) -> Result<()> {
+		// KEYMAP UPDATES
+		{
+			let mut old_keymap_id = self.current_keymap_id.lock().await;
+
+			if *old_keymap_id != keymap_id {
+				let keymap_key = DefaultKey::from(KeyData::from_ffi(keymap_id));
+
+				// Get keymap data and drop the lock immediately
+				let keymap_data = {
+					let keymap_lock = KEYMAPS.lock();
+					keymap_lock
+						.get(keymap_key)
+						.map(|s| s.as_bytes().to_vec())
+						.unwrap_or_default()
+				};
+
+				// Now we can safely await
+				self.send_keymap(client, &keymap_data).await?;
+			};
+			*old_keymap_id = keymap_id;
+			drop(old_keymap_id);
+		}
+
+		// PRESSED KEYS UPDATE
+		let pressed_keys = self.pressed_keys.entry(surface.id).or_default();
+		if pressed {
+			pressed_keys.insert(key);
+		} else {
+			pressed_keys.remove(&key);
+		}
+		// println!("pressed keys: {:?}", &*pressed_keys);
+
+		// FOCUS UPDATES
+		let mut focused = self.focused_surface.lock().await;
+		let mut modifier_state = self.modifier_state.lock().await;
+
+		let refocus = focused.as_ptr() != Arc::as_ptr(&surface);
+		// If we're entering a new surface
+		if refocus {
+			// Send leave to old surface if it exists and is still alive
+			if let Some(old_surface) = focused.upgrade() {
+				let serial = client.next_event_serial();
+				self.leave(client, old_surface.id, serial, self.id).await?;
+				// println!("Left surface {}", old_surface.id);
+			}
+
+			// Send enter to new surface
+			let serial = client.next_event_serial();
+			self.enter(
+				client,
+				self.id,
+				serial,
+				surface.id,
+				pressed_keys.iter().flat_map(|k| k.to_ne_bytes()).collect(),
+			)
+			.await?;
+			// println!("Entered new surface {}", surface.id);
+
+			// Update focused surface
+			*focused = Arc::downgrade(&surface);
+		}
+
+		// KEY EVENT SENDING
+		let serial = client.next_event_serial();
+		// println!(
+		// 	"Sent key {key} {}",
+		// 	if pressed { "pressed" } else { "released" }
+		// );
+		self.key(
+			client,
+			self.id,
+			serial,
+			client.display().creation_time.elapsed().as_millis() as u32, // time
+			key,
+			if pressed {
+				KeyState::Pressed
+			} else {
+				KeyState::Released
+			},
+		)
+		.await?;
+
+		// MODIFIER UPDATES
+		// Update modifier state and send modifiers event if changed
+		if refocus || modifier_state.update_key(key, pressed) {
+			// println!("Update modifiers");
+			let serial = client.next_event_serial();
+			self.modifiers(
+				client,
+				self.id,
+				serial,
+				modifier_state.mods_depressed,
+				modifier_state.mods_latched,
+				modifier_state.mods_locked,
+				modifier_state.group,
+			)
+			.await?;
+		}
+
+		Ok(())
+	}
+
+	pub async fn reset(&self, client: &mut Client) -> Result<()> {
+		let mut modifier_state = self.modifier_state.lock().await;
+		modifier_state.pressed_keys.clear();
+		modifier_state.mods_depressed = 0;
+		modifier_state.mods_latched = 0;
+		modifier_state.mods_locked = 0;
+		modifier_state.group = 0;
+
+		let serial = client.next_event_serial();
+		self.modifiers(
+			client,
+			self.id,
+			serial,
+			modifier_state.mods_depressed,
+			modifier_state.mods_latched,
+			modifier_state.mods_locked,
+			modifier_state.group,
+		)
+		.await
+	}
+}
+
+impl WlKeyboard for Keyboard {
+	/// https://wayland.app/protocols/wayland#wl_keyboard:request:release
+	async fn release(&self, _client: &mut Client, _sender_id: ObjectId) -> Result<()> {
+		Ok(())
+	}
+}

src/wayland/core/mod.rs

@@ -0,0 +1,13 @@
+pub mod buffer;
+pub mod callback;
+pub mod compositor;
+pub mod data_device;
+pub mod keyboard;
+pub mod output;
+pub mod pointer;
+pub mod seat;
+pub mod shm;
+pub mod shm_buffer_backing;
+pub mod shm_pool;
+pub mod surface;
+pub mod touch;

src/wayland/core/output.rs

@@ -0,0 +1,43 @@
+use waynest::{
+	server::{Client, Dispatcher, Result},
+	wire::ObjectId,
+};
+
+pub use waynest::server::protocol::core::wayland::wl_output::*;
+
+#[derive(Debug, Dispatcher)]
+pub struct Output {
+	pub id: ObjectId,
+	pub version: u32,
+}
+impl Output {
+	pub async fn advertise_outputs(&self, client: &mut Client) -> Result<()> {
+		self.geometry(
+			client,
+			self.id,
+			2048,
+			2048,
+			0,
+			0,
+			Subpixel::None,
+			"Stardust Virtual Display".to_string(),
+			"Stardust Virtual Display".to_string(),
+			Transform::Normal,
+		)
+		.await?;
+
+		self.mode(client, self.id, Mode::Current, 2048, 2048, i32::MAX)
+			.await?;
+
+		if self.version >= 2 {
+			self.done(client, self.id).await?;
+		}
+		Ok(())
+	}
+}
+impl WlOutput for Output {
+	/// https://wayland.app/protocols/wayland#wl_output:request:release
+	async fn release(&self, _client: &mut Client, _sender_id: ObjectId) -> Result<()> {
+		Ok(())
+	}
+}

src/wayland/core/pointer.rs

@@ -0,0 +1,190 @@
+use crate::wayland::core::{seat::fixed_from_f32, surface::Surface};
+use mint::Vector2;
+use std::sync::Arc;
+use std::sync::Weak;
+use tokio::sync::Mutex;
+use tracing;
+pub use waynest::server::protocol::core::wayland::wl_pointer::*;
+use waynest::{
+	server::{Client, Dispatcher, Result},
+	wire::ObjectId,
+};
+
+#[derive(Dispatcher)]
+pub struct Pointer {
+	pub id: ObjectId,
+	version: u32,
+	focused_surface: Mutex<Weak<Surface>>,
+}
+impl Pointer {
+	pub fn new(id: ObjectId, version: u32) -> Self {
+		Self {
+			id,
+			version,
+			focused_surface: Mutex::new(Weak::new()),
+		}
+	}
+
+	pub async fn handle_pointer_motion(
+		&self,
+		client: &mut Client,
+		surface: Arc<Surface>,
+		position: Vector2<f32>,
+	) -> Result<()> {
+		tracing::debug!(
+			"Handling pointer motion at ({}, {})",
+			position.x,
+			position.y
+		);
+		let mut focused = self.focused_surface.lock().await;
+
+		// If we're entering a new surface
+		if focused.as_ptr() != Arc::as_ptr(&surface) {
+			tracing::debug!("Surface transition detected");
+			// Send leave to old surface if it exists and is still alive
+			if let Some(old_surface) = focused.upgrade() {
+				let serial = client.next_event_serial();
+				tracing::debug!("Sending leave event with serial {}", serial);
+				self.leave(client, self.id, serial, old_surface.id).await?;
+			}
+
+			// Send enter to new surface
+			let serial = client.next_event_serial();
+			tracing::debug!(
+				"Sending enter event with serial {} to surface {:?}",
+				serial,
+				surface.id
+			);
+			self.enter(
+				client,
+				self.id,
+				serial,
+				surface.id,
+				fixed_from_f32(position.x),
+				fixed_from_f32(position.y),
+			)
+			.await?;
+
+			// Update focused surface
+			*focused = Arc::downgrade(&surface);
+		}
+
+		// Send motion event to current surface
+		tracing::debug!("Sending motion event to surface");
+		self.motion(
+			client,
+			self.id,
+			0, // time
+			fixed_from_f32(position.x),
+			fixed_from_f32(position.y),
+		)
+		.await?;
+		if self.version >= 5 {
+			self.frame(client, self.id).await?;
+		}
+
+		Ok(())
+	}
+
+	pub async fn handle_pointer_button(
+		&self,
+		client: &mut Client,
+		surface: Arc<Surface>,
+		button: u32,
+		pressed: bool,
+	) -> Result<()> {
+		tracing::debug!(
+			"Handling pointer button {} {} on surface {:?}",
+			button,
+			if pressed { "pressed" } else { "released" },
+			surface.id
+		);
+		let serial = client.next_event_serial();
+		self.button(
+			client,
+			self.id,
+			serial,
+			0, // time
+			button,
+			if pressed {
+				ButtonState::Pressed
+			} else {
+				ButtonState::Released
+			},
+		)
+		.await?;
+		self.frame(client, self.id).await
+	}
+	pub async fn handle_pointer_scroll(
+		&self,
+		client: &mut Client,
+		_surface: Arc<Surface>,
+		scroll_distance: Option<Vector2<f32>>,
+		scroll_steps: Option<Vector2<f32>>,
+	) -> Result<()> {
+		tracing::debug!(
+			"Handling pointer scroll: distance={:?}, steps={:?}",
+			scroll_distance,
+			scroll_steps
+		);
+		if let Some(distance) = scroll_distance {
+			self.axis(
+				client,
+				self.id,
+				0, // time
+				Axis::HorizontalScroll,
+				fixed_from_f32(distance.x),
+			)
+			.await?;
+			self.axis(
+				client,
+				self.id,
+				0, // time
+				Axis::VerticalScroll,
+				fixed_from_f32(distance.y),
+			)
+			.await?;
+		}
+		if self.version >= 5 {
+			if let Some(steps) = scroll_steps {
+				self.axis_discrete(client, self.id, Axis::HorizontalScroll, steps.x as i32)
+					.await?;
+				self.axis_discrete(client, self.id, Axis::VerticalScroll, steps.y as i32)
+					.await?;
+			}
+			self.frame(client, self.id).await?;
+		}
+		Ok(())
+	}
+
+	pub async fn reset(&self, client: &mut Client) -> Result<()> {
+		let mut focused = self.focused_surface.lock().await;
+		if let Some(old_surface) = focused.upgrade() {
+			let serial = client.next_event_serial();
+			self.leave(client, self.id, serial, old_surface.id).await?;
+			self.frame(client, self.id).await?;
+		}
+		*focused = Weak::new();
+		Ok(())
+	}
+}
+
+impl WlPointer for Pointer {
+	/// https://wayland.app/protocols/wayland#wl_pointer:request:set_cursor
+	async fn set_cursor(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_serial: u32,
+		_surface: Option<ObjectId>,
+		_hotspot_x: i32,
+		_hotspot_y: i32,
+	) -> Result<()> {
+		Ok(())
+	}
+
+	/// https://wayland.app/protocols/wayland#wl_pointer:request:release
+	async fn release(&self, _client: &mut Client, _sender_id: ObjectId) -> Result<()> {
+		Ok(())
+	}
+}

src/wayland/core/seat.rs

@@ -0,0 +1,200 @@
+use crate::wayland::core::{keyboard::Keyboard, pointer::Pointer, surface::Surface, touch::Touch};
+use mint::Vector2;
+use std::sync::Arc;
+use std::sync::OnceLock;
+pub use waynest::server::protocol::core::wayland::wl_seat::*;
+use waynest::server::{Client, Dispatcher, Result};
+use waynest::wire::{Fixed, ObjectId};
+
+#[derive(Debug)]
+pub enum SeatMessage {
+	PointerMotion {
+		surface: Arc<Surface>,
+		position: Vector2<f32>,
+	},
+	PointerButton {
+		surface: Arc<Surface>,
+		button: u32,
+		pressed: bool,
+	},
+	PointerScroll {
+		surface: Arc<Surface>,
+		scroll_distance: Option<Vector2<f32>>,
+		scroll_steps: Option<Vector2<f32>>,
+	},
+	KeyboardKey {
+		surface: Arc<Surface>,
+		keymap_id: u64,
+		key: u32,
+		pressed: bool,
+	},
+	TouchDown {
+		surface: Arc<Surface>,
+		id: u32,
+		position: Vector2<f32>,
+	},
+	TouchMove {
+		id: u32,
+		position: Vector2<f32>,
+	},
+	TouchUp {
+		id: u32,
+	},
+	Reset,
+}
+
+pub fn fixed_from_f32(f: f32) -> Fixed {
+	unsafe { Fixed::from_raw((f * 256.0).round() as u32) }
+}
+
+#[derive(Default, Dispatcher)]
+pub struct Seat {
+	version: u32,
+	pointer: OnceLock<Arc<Pointer>>,
+	keyboard: OnceLock<Arc<Keyboard>>,
+	touch: OnceLock<Arc<Touch>>,
+}
+
+impl Seat {
+	pub async fn new(client: &mut Client, id: ObjectId, version: u32) -> Result<Self> {
+		let seat = Self {
+			version,
+			pointer: OnceLock::new(),
+			keyboard: OnceLock::new(),
+			touch: OnceLock::new(),
+		};
+
+		if version >= 2 {
+			seat.name(client, id, "theonlyseat".into()).await?;
+		}
+
+		tracing::debug!("Advertising seat capabilities with id {}", id);
+		let capabilities = Capability::Pointer | Capability::Keyboard | Capability::Touch;
+		WlSeat::capabilities(&seat, client, id, capabilities).await?;
+		tracing::debug!("Capabilities advertised: {:?}", capabilities);
+
+		Ok(seat)
+	}
+
+	pub async fn handle_message(&self, client: &mut Client, message: SeatMessage) -> Result<()> {
+		match message {
+			SeatMessage::PointerMotion { surface, position } => {
+				if let Some(pointer) = self.pointer.get() {
+					pointer
+						.handle_pointer_motion(client, surface, position)
+						.await?;
+				}
+			}
+			SeatMessage::PointerButton {
+				surface,
+				button,
+				pressed,
+			} => {
+				if let Some(pointer) = self.pointer.get() {
+					pointer
+						.handle_pointer_button(client, surface, button, pressed)
+						.await?;
+				}
+			}
+			SeatMessage::PointerScroll {
+				surface,
+				scroll_distance,
+				scroll_steps,
+			} => {
+				if let Some(pointer) = self.pointer.get() {
+					pointer
+						.handle_pointer_scroll(client, surface, scroll_distance, scroll_steps)
+						.await?;
+				}
+			}
+			SeatMessage::KeyboardKey {
+				surface,
+				keymap_id,
+				key,
+				pressed,
+			} => {
+				if let Some(keyboard) = self.keyboard.get() {
+					keyboard
+						.handle_keyboard_key(client, surface, keymap_id, key - 8, pressed)
+						.await?;
+				}
+			}
+			SeatMessage::TouchDown {
+				surface,
+				id,
+				position,
+			} => {
+				if let Some(touch) = self.touch.get() {
+					touch
+						.handle_touch_down(client, surface, id, position)
+						.await?;
+				}
+			}
+			SeatMessage::TouchMove { id, position } => {
+				if let Some(touch) = self.touch.get() {
+					touch.handle_touch_move(client, id, position).await?;
+				}
+			}
+			SeatMessage::TouchUp { id } => {
+				if let Some(touch) = self.touch.get() {
+					touch.handle_touch_up(client, id).await?;
+				}
+			}
+			SeatMessage::Reset => {
+				if let Some(pointer) = self.pointer.get() {
+					pointer.reset(client).await?;
+				}
+				if let Some(keyboard) = self.keyboard.get() {
+					keyboard.reset(client).await?;
+				}
+				if let Some(touch) = self.touch.get() {
+					touch.reset(client).await?;
+				}
+			}
+		}
+		Ok(())
+	}
+}
+impl WlSeat for Seat {
+	/// https://wayland.app/protocols/wayland#wl_seat:request:get_pointer
+	async fn get_pointer(
+		&self,
+		client: &mut Client,
+		_sender_id: ObjectId,
+		id: ObjectId,
+	) -> Result<()> {
+		let pointer = client.insert(id, Pointer::new(id, self.version));
+		let _ = self.pointer.set(pointer);
+		Ok(())
+	}
+
+	/// https://wayland.app/protocols/wayland#wl_seat:request:get_keyboard
+	async fn get_keyboard(
+		&self,
+		client: &mut Client,
+		_sender_id: ObjectId,
+		id: ObjectId,
+	) -> Result<()> {
+		tracing::info!("Getting keyboard");
+		let keyboard = client.insert(id, Keyboard::new(id));
+		let _ = self.keyboard.set(keyboard);
+		Ok(())
+	}
+
+	/// https://wayland.app/protocols/wayland#wl_seat:request:get_touch
+	async fn get_touch(
+		&self,
+		client: &mut Client,
+		_sender_id: ObjectId,
+		id: ObjectId,
+	) -> Result<()> {
+		let touch = client.insert(id, Touch(id));
+		let _ = self.touch.set(touch);
+		Ok(())
+	}
+
+	/// https://wayland.app/protocols/wayland#wl_seat:request:release
+	async fn release(&self, _client: &mut Client, _sender_id: ObjectId) -> Result<()> {
+		Ok(())
+	}
+}

src/wayland/core/shm.rs

@@ -0,0 +1,39 @@
+use std::os::fd::OwnedFd;
+
+use crate::wayland::core::shm_pool::ShmPool;
+pub use waynest::server::protocol::core::wayland::wl_shm::*;
+use waynest::{
+	server::{Client, Dispatcher, Result},
+	wire::ObjectId,
+};
+
+#[derive(Debug, Dispatcher, Default)]
+pub struct Shm;
+impl Shm {
+	pub async fn advertise_formats(&self, client: &mut Client, sender_id: ObjectId) -> Result<()> {
+		self.format(client, sender_id, Format::Argb8888).await?;
+		self.format(client, sender_id, Format::Xrgb8888).await?;
+
+		Ok(())
+	}
+}
+impl WlShm for Shm {
+	/// https://wayland.app/protocols/wayland#wl_shm:request:create_pool
+	async fn create_pool(
+		&self,
+		client: &mut Client,
+		_sender_id: ObjectId,
+		pool_id: ObjectId,
+		fd: OwnedFd,
+		size: i32,
+	) -> Result<()> {
+		client.insert(pool_id, ShmPool::new(fd, size)?);
+
+		Ok(())
+	}
+
+	/// https://wayland.app/protocols/wayland#wl_shm:request:release
+	async fn release(&self, _client: &mut Client, _sender_id: ObjectId) -> Result<()> {
+		Ok(())
+	}
+}

src/wayland/core/shm_buffer_backing.rs

@@ -0,0 +1,99 @@
+use super::shm_pool::ShmPool;
+use bevy::{
+	asset::{Assets, Handle, RenderAssetUsages},
+	image::Image,
+	render::render_resource::{Extent3d, TextureDimension, TextureFormat},
+};
+use mint::Vector2;
+use std::sync::Arc;
+use waynest::server::protocol::core::wayland::wl_shm::Format;
+/// Parameters for a shared memory buffer
+#[derive(Debug)]
+pub struct ShmBufferBacking {
+	pool: Arc<ShmPool>,
+	offset: usize,
+	stride: usize,
+	size: Vector2<usize>,
+	format: Format,
+}
+
+impl ShmBufferBacking {
+	pub fn new(
+		pool: Arc<ShmPool>,
+		offset: usize,
+		stride: usize,
+		size: Vector2<usize>,
+		format: Format,
+	) -> Self {
+		Self {
+			pool,
+			offset,
+			stride,
+			size,
+			format,
+		}
+	}
+
+	#[tracing::instrument("debug", skip_all)]
+	pub fn update_tex(&self, images: &mut Assets<Image>) -> Option<Handle<Image>> {
+		let src_data_lock = self.pool.data_lock();
+		let mut src_cursor = self.offset;
+
+		// Calculate maximum cursor position needed - stride is already in bytes
+		let max_cursor = self.offset + (self.size.y * self.stride);
+
+		// Check if we have enough data
+		if max_cursor > src_data_lock.len() {
+			return None;
+		}
+		let data_len = self.size.x * self.size.y * 4;
+		if src_data_lock.len() != data_len {
+			return None;
+		}
+		let mut dst_cursor = 0;
+
+		let mut dst_data = vec![0u8; data_len];
+		for _y in 0..self.size.y {
+			for _x in 0..self.size.x {
+				match self.format {
+					Format::Argb8888 | Format::Xrgb8888 => {
+						dst_data[dst_cursor] = src_data_lock[src_cursor + 2]; // Red is byte 2
+						dst_data[dst_cursor + 1] = src_data_lock[src_cursor + 1]; // Green is byte 1
+						dst_data[dst_cursor + 2] = src_data_lock[src_cursor]; // Blue is byte 0
+						dst_data[dst_cursor + 3] = src_data_lock[src_cursor + 3]; // Alpha is byte 3
+					}
+					_ => panic!("Unsupported format {:?}", self.format),
+				}
+				src_cursor += 4;
+				dst_cursor += 4;
+			}
+			src_cursor += self.stride - (self.size.x * 4);
+		}
+
+		let image = Image::new(
+			Extent3d {
+				width: self.size().x as u32,
+				height: self.size().y as u32,
+				depth_or_array_layers: 1,
+			},
+			TextureDimension::D2,
+			dst_data,
+			TextureFormat::Rgba8UnormSrgb,
+			RenderAssetUsages::RENDER_WORLD,
+		);
+
+		Some(images.add(image))
+	}
+
+	pub fn is_transparent(&self) -> bool {
+		match self.format {
+			Format::Xrgb8888 => false,
+			Format::Argb8888 => true,
+			_ => true,
+		}
+	}
+
+	pub fn size(&self) -> Vector2<usize> {
+		self.size
+	}
+}

src/wayland/core/shm_pool.rs

@@ -0,0 +1,79 @@
+use memmap2::{MmapOptions, RemapOptions};
+use parking_lot::{Mutex, MutexGuard, RawMutex, lock_api::MappedMutexGuard};
+use std::os::fd::{IntoRawFd, OwnedFd};
+use waynest::{
+	server::{Client, Dispatcher, Result, protocol::core::wayland::wl_shm::Format},
+	wire::ObjectId,
+};
+
+use crate::wayland::core::buffer::{Buffer, BufferBacking};
+
+pub use waynest::server::protocol::core::wayland::wl_shm_pool::*;
+
+use super::shm_buffer_backing::ShmBufferBacking;
+
+#[derive(Debug, Dispatcher)]
+pub struct ShmPool {
+	inner: Mutex<memmap2::MmapMut>,
+}
+
+impl ShmPool {
+	#[tracing::instrument(level = "debug", skip_all)]
+	pub fn new(fd: OwnedFd, size: i32) -> Result<Self> {
+		let map = unsafe {
+			MmapOptions::new()
+				.len(size as usize)
+				.map_mut(fd.into_raw_fd())?
+		};
+
+		Ok(Self {
+			inner: Mutex::new(map),
+		})
+	}
+
+	#[tracing::instrument(level = "debug", skip_all)]
+	pub fn data_lock(&self) -> MappedMutexGuard<RawMutex, [u8]> {
+		MutexGuard::map(self.inner.lock(), |i| i.as_mut())
+	}
+}
+
+impl WlShmPool for ShmPool {
+	/// https://wayland.app/protocols/wayland#wl_shm_pool:request:create_buffer
+	#[tracing::instrument(level = "debug", skip_all)]
+	async fn create_buffer(
+		&self,
+		client: &mut Client,
+		sender_id: ObjectId,
+		id: ObjectId,
+		offset: i32,
+		width: i32,
+		height: i32,
+		stride: i32,
+		format: Format,
+	) -> Result<()> {
+		let params = ShmBufferBacking::new(
+			client.get::<ShmPool>(sender_id).unwrap(),
+			offset as usize,
+			stride as usize,
+			[width as usize, height as usize].into(),
+			format,
+		);
+
+		Buffer::new(client, id, BufferBacking::Shm(params));
+		Ok(())
+	}
+
+	/// https://wayland.app/protocols/wayland#wl_shm_pool:request:resize
+	#[tracing::instrument(level = "debug", skip_all)]
+	async fn resize(&self, _client: &mut Client, _sender_id: ObjectId, size: i32) -> Result<()> {
+		let mut inner = self.inner.lock();
+		unsafe { inner.remap(size as usize, RemapOptions::new().may_move(true))? };
+		Ok(())
+	}
+
+	/// https://wayland.app/protocols/wayland#wl_shm_pool:request:destroy
+	#[tracing::instrument(level = "debug", skip_all)]
+	async fn destroy(&self, _client: &mut Client, _sender_id: ObjectId) -> Result<()> {
+		Ok(())
+	}
+}

src/wayland/core/surface.rs

@@ -0,0 +1,424 @@
+use super::{buffer::Buffer, callback::Callback};
+use crate::{
+	BevyMaterial,
+	core::registry::Registry,
+	nodes::{drawable::model::ModelPart, items::panel::Geometry},
+	wayland::{
+		Message, MessageSink,
+		core::buffer::BufferUsage,
+		presentation::{MonotonicTimestamp, PresentationFeedback},
+		util::{ClientExt, DoubleBuffer},
+		xdg::{popup::Popup, toplevel::Toplevel},
+	},
+};
+use bevy::{
+	asset::{Assets, Handle},
+	image::Image,
+	render::alpha::AlphaMode,
+};
+use bevy_dmabuf::import::ImportedDmatexs;
+use mint::Vector2;
+use parking_lot::Mutex;
+use std::sync::{Arc, OnceLock};
+use waynest::{
+	server::{
+		Client, Dispatcher, Result,
+		protocol::{
+			core::wayland::{wl_output::Transform, wl_surface::*},
+			stable::presentation_time::wp_presentation_feedback::{Kind, WpPresentationFeedback},
+		},
+	},
+	wire::ObjectId,
+};
+
+pub static WL_SURFACE_REGISTRY: Registry<Surface> = Registry::new();
+
+#[derive(Debug, Clone)]
+pub enum SurfaceRole {
+	XdgToplevel(Arc<Toplevel>),
+	XDGPopup(Arc<Popup>),
+}
+
+#[derive(Debug, Clone)]
+pub struct BufferState {
+	pub buffer: Arc<Buffer>,
+	pub usage: Option<Arc<BufferUsage>>,
+}
+
+#[derive(Debug, Clone)]
+pub struct SurfaceState {
+	pub buffer: Option<BufferState>,
+	pub density: f32,
+	pub geometry: Option<Geometry>,
+	pub min_size: Option<Vector2<u32>>,
+	pub max_size: Option<Vector2<u32>>,
+	frame_callbacks: Vec<Arc<Callback>>,
+}
+impl Default for SurfaceState {
+	fn default() -> Self {
+		Self {
+			buffer: Default::default(),
+			density: 1.0,
+			geometry: None,
+			min_size: None,
+			max_size: None,
+			frame_callbacks: Vec::new(),
+		}
+	}
+}
+
+// if returning false, don't run this callback again... just remove it
+pub type OnCommitCallback = Box<dyn Fn(&Surface, &SurfaceState) -> bool + Send + Sync>;
+
+#[derive(Dispatcher)]
+pub struct Surface {
+	pub id: ObjectId,
+	state: Mutex<DoubleBuffer<SurfaceState>>,
+	pub message_sink: MessageSink,
+	// TODO: This should probably be a OnceLock? wayland doesn't support changing the surface role
+	pub role: Mutex<Option<SurfaceRole>>,
+	on_commit_handlers: Mutex<Vec<OnCommitCallback>>,
+	material: OnceLock<Handle<BevyMaterial>>,
+	pending_material_applications: Registry<ModelPart>,
+	presentation_feedback: Mutex<Vec<Arc<PresentationFeedback>>>,
+}
+impl std::fmt::Debug for Surface {
+	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+		f.debug_struct("Surface")
+			.field("state", &self.state)
+			.field("message_sink", &self.message_sink)
+			.field("role", &self.role)
+			.field(
+				"on_commit_handlers",
+				&format!("<{} handlers>", self.on_commit_handlers.lock().len()),
+			)
+			.finish()
+	}
+}
+impl Surface {
+	#[tracing::instrument(level = "debug", skip_all)]
+	pub fn new(client: &Client, id: ObjectId) -> Self {
+		Surface {
+			id,
+			state: Default::default(),
+			message_sink: client.message_sink(),
+			role: Mutex::new(None),
+			on_commit_handlers: Mutex::new(Vec::new()),
+			material: OnceLock::new(),
+			pending_material_applications: Registry::new(),
+			presentation_feedback: Mutex::default(),
+		}
+	}
+
+	#[tracing::instrument(level = "debug", skip_all)]
+	pub fn pending_state(&self) -> parking_lot::MutexGuard<'_, DoubleBuffer<SurfaceState>> {
+		self.state.lock()
+	}
+
+	#[tracing::instrument(level = "debug", skip_all)]
+	pub fn add_commit_handler<F: Fn(&Surface, &SurfaceState) -> bool + Send + Sync + 'static>(
+		&self,
+		handler: F,
+	) {
+		let mut handlers = self.on_commit_handlers.lock();
+		handlers.push(Box::new(handler));
+	}
+
+	#[tracing::instrument(level = "debug", skip_all)]
+	pub fn update_graphics(
+		&self,
+		dmatexes: &ImportedDmatexs,
+		materials: &mut Assets<BevyMaterial>,
+		images: &mut Assets<Image>,
+	) {
+		let Some(buffer) = self.state.lock().current().buffer.clone() else {
+			return;
+		};
+
+		let material = self.material.get_or_init(|| {
+			// // Set default shader parameters
+			// let mut params = mat_wrapper.0.get_all_param_info();
+			// params.set_vec2("uv_scale", stereokit_rust::maths::Vec2::new(1.0, 1.0));
+			// params.set_vec2("uv_offset", stereokit_rust::maths::Vec2::new(0.0, 0.0));
+			// params.set_float("fcFactor", 1.0);
+			// params.set_float("ripple", 4.0);
+			// params.set_float("alpha_min", 0.0);
+			// params.set_float("alpha_max", 1.0);
+
+			materials.add(BevyMaterial {
+				unlit: true,
+				..Default::default()
+			})
+		});
+
+		if let Some(new_tex) = buffer.buffer.update_tex(dmatexes, images) {
+			let material = materials.get_mut(material).unwrap();
+			material.base_color_texture.replace(new_tex);
+			material.alpha_mode = if buffer.buffer.is_transparent() {
+				AlphaMode::Premultiplied
+			} else {
+				AlphaMode::Opaque
+			};
+		}
+
+		self.apply_surface_materials();
+	}
+
+	#[tracing::instrument(level = "debug", skip_all)]
+	pub fn apply_material(&self, model_part: &Arc<ModelPart>) {
+		// tracing::info!("uwu applying material");
+		self.pending_material_applications.add_raw(model_part)
+	}
+
+	#[tracing::instrument(level = "debug", skip_all)]
+	fn apply_surface_materials(&self) {
+		let Some(mat) = self.material.get() else {
+			return;
+		};
+
+		for model_node in self.pending_material_applications.get_valid_contents() {
+			model_node.replace_material(mat.clone());
+		}
+		self.pending_material_applications.clear();
+	}
+	#[tracing::instrument("debug", skip_all)]
+	pub fn current_state(&self) -> SurfaceState {
+		self.state.lock().current().clone()
+	}
+	#[tracing::instrument(level = "debug", skip_all)]
+	pub fn frame_event(&self) {
+		for callback in self.state.lock().current.frame_callbacks.drain(..) {
+			let _ = self.message_sink.send(Message::Frame(callback));
+		}
+	}
+	// pub fn size(&self) -> Option<Vector2<u32>> {
+	// 	self.state
+	// 		.lock()
+	// 		.current()
+	// 		.buffer
+	// 		.as_ref()
+	// 		.map(|b| [b.size.x as u32, b.size.y as u32].into())
+	// }
+
+	// pub async fn release_old_buffer(&self, client: &mut Client) -> Result<()> {
+	// 	let (old_buffer, object) = {
+	// 		let lock = self.state.lock();
+
+	// 		let Some(old_buffer) = lock.current().buffer.clone() else {
+	// 			return Ok(());
+	// 		};
+	// 		let new_buffer = lock.pending.buffer.as_ref();
+	// 		if new_buffer.map(Arc::as_ptr) == Some(Arc::as_ptr(&old_buffer)) {
+	// 			return Ok(());
+	// 		}
+	// 		drop(lock);
+
+	// 		(old_buffer.clone(), old_buffer.id)
+	// 	};
+	// 	old_buffer.release(client, object).await?;
+
+	// 	Ok(())
+	// }
+
+	#[tracing::instrument(level = "debug", skip_all)]
+	pub fn add_presentation_feedback(&self, feedback: Arc<PresentationFeedback>) {
+		self.presentation_feedback.lock().push(feedback);
+	}
+
+	pub fn submit_presentation_feedback(
+		self: &Arc<Self>,
+		display_timestamp: MonotonicTimestamp,
+		refresh_cycle: u64,
+	) {
+		let _ = self.message_sink.send(Message::SendPresentationFeedback {
+			surface: self.clone(),
+			display_timestamp,
+			refresh_cycle,
+		});
+	}
+
+	#[tracing::instrument(level = "debug", skip_all)]
+	pub async fn send_presentation_feedback(
+		&self,
+		client: &mut Client,
+		display_timestamp: MonotonicTimestamp,
+		refresh_cycle: u64,
+	) -> Result<()> {
+		let feedbacks = self
+			.presentation_feedback
+			.lock()
+			.drain(..)
+			.collect::<Vec<_>>();
+		for feedback in feedbacks {
+			feedback
+				.sync_output(
+					client,
+					feedback.0,
+					client.display().output.get().unwrap().id,
+				)
+				.await?;
+			let cycle_lo = refresh_cycle as u32;
+			let cycle_hi = (refresh_cycle >> 32) as u32;
+			feedback
+				.presented(
+					client,
+					feedback.0,
+					display_timestamp.secs_hi(),
+					display_timestamp.secs_lo(),
+					display_timestamp.subsec_nanos(),
+					0,
+					cycle_hi,
+					cycle_lo,
+					Kind::empty(),
+				)
+				.await?;
+		}
+		Ok(())
+	}
+}
+impl WlSurface for Surface {
+	/// https://wayland.app/protocols/wayland#wl_surface:request:attach
+	#[tracing::instrument(level = "debug", skip_all)]
+	async fn attach(
+		&self,
+		client: &mut Client,
+		_sender_id: ObjectId,
+		buffer: Option<ObjectId>,
+		_x: i32,
+		_y: i32,
+	) -> Result<()> {
+		self.state.lock().pending.buffer = buffer.and_then(|b| {
+			let buffer = client.get::<Buffer>(b)?;
+			let mut usage = Some(BufferUsage::new(client, &buffer));
+			Some(BufferState {
+				usage: usage.take_if(|_| buffer.uses_buffer_usage()),
+				buffer,
+			})
+		});
+		Ok(())
+	}
+
+	/// https://wayland.app/protocols/wayland#wl_surface:request:damage
+	#[tracing::instrument(level = "debug", skip_all)]
+	async fn damage(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_x: i32,
+		_y: i32,
+		_width: i32,
+		_height: i32,
+	) -> Result<()> {
+		Ok(())
+	}
+
+	/// https://wayland.app/protocols/wayland#wl_surface:request:frame
+	#[tracing::instrument(level = "debug", skip_all)]
+	async fn frame(
+		&self,
+		client: &mut Client,
+		_sender_id: ObjectId,
+		callback_id: ObjectId,
+	) -> Result<()> {
+		let callback = client.insert(callback_id, Callback(callback_id));
+		self.state.lock().pending.frame_callbacks.push(callback);
+		Ok(())
+	}
+
+	/// https://wayland.app/protocols/wayland#wl_surface:request:set_opaque_region
+	#[tracing::instrument(level = "debug", skip_all)]
+	async fn set_opaque_region(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_region: Option<ObjectId>,
+	) -> Result<()> {
+		// nothing we can really do to repaint behind this so ignore it
+		Ok(())
+	}
+
+	/// https://wayland.app/protocols/wayland#wl_surface:request:set_input_region
+	#[tracing::instrument(level = "debug", skip_all)]
+	async fn set_input_region(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_region: Option<ObjectId>,
+	) -> Result<()> {
+		// too complicated to implement this for now so who the hell cares
+		Ok(())
+	}
+
+	/// https://wayland.app/protocols/wayland#wl_surface:request:commit
+	#[tracing::instrument(level = "debug", skip_all)]
+	async fn commit(&self, _client: &mut Client, _sender_id: ObjectId) -> Result<()> {
+		self.state.lock().apply();
+		self.state.lock().pending.frame_callbacks.clear();
+		let current_state = self.current_state();
+		let mut handlers = self.on_commit_handlers.lock();
+		handlers.retain(|f| (f)(self, &current_state));
+		Ok(())
+	}
+
+	/// https://wayland.app/protocols/wayland#wl_surface:request:set_buffer_transform
+	#[tracing::instrument(level = "debug", skip_all)]
+	async fn set_buffer_transform(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_transform: Transform,
+	) -> Result<()> {
+		// we just don't have the output transform or fullscreen at all so this optimization is never needed
+		Ok(())
+	}
+
+	/// https://wayland.app/protocols/wayland#wl_surface:request:set_buffer_scale
+	#[tracing::instrument(level = "debug", skip_all)]
+	async fn set_buffer_scale(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		scale: i32,
+	) -> Result<()> {
+		self.state.lock().pending.density = scale as f32;
+		Ok(())
+	}
+
+	/// https://wayland.app/protocols/wayland#wl_surface:request:damage_buffer
+	#[tracing::instrument(level = "debug", skip_all)]
+	async fn damage_buffer(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_x: i32,
+		_y: i32,
+		_width: i32,
+		_height: i32,
+	) -> Result<()> {
+		Ok(())
+	}
+
+	/// https://wayland.app/protocols/wayland#wl_surface:request:offset
+	#[tracing::instrument(level = "debug", skip_all)]
+	async fn offset(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		_x: i32,
+		_y: i32,
+	) -> Result<()> {
+		Ok(())
+	}
+
+	/// https://wayland.app/protocols/wayland#wl_surface:request:destroy
+	#[tracing::instrument(level = "debug", skip_all)]
+	async fn destroy(&self, _client: &mut Client, _sender_id: ObjectId) -> Result<()> {
+		Ok(())
+	}
+}
+
+impl Drop for Surface {
+	fn drop(&mut self) {
+		self.role.lock().take();
+	}
+}

src/wayland/core/touch.rs

@@ -0,0 +1,75 @@
+use crate::wayland::core::surface::Surface;
+use mint::Vector2;
+use std::sync::Arc;
+pub use waynest::server::protocol::core::wayland::wl_touch::*;
+use waynest::{
+	server::{Client, Dispatcher, Result},
+	wire::ObjectId,
+};
+
+use super::seat::fixed_from_f32;
+
+#[derive(Debug, Dispatcher)]
+pub struct Touch(pub ObjectId);
+impl Touch {
+	pub async fn handle_touch_down(
+		&self,
+		client: &mut Client,
+		surface: Arc<Surface>,
+		id: u32,
+		position: Vector2<f32>,
+	) -> Result<()> {
+		let serial = client.next_event_serial();
+		self.down(
+			client,
+			self.0,
+			serial,
+			0,
+			surface.id,
+			id as i32,
+			fixed_from_f32(position.x),
+			fixed_from_f32(position.y),
+		)
+		.await?;
+		self.frame(client, self.0).await
+	}
+
+	pub async fn handle_touch_move(
+		&self,
+		client: &mut Client,
+		id: u32,
+		position: Vector2<f32>,
+	) -> Result<()> {
+		self.motion(
+			client,
+			self.0,
+			0,
+			id as i32,
+			fixed_from_f32(position.x),
+			fixed_from_f32(position.y),
+		)
+		.await?;
+		self.frame(client, self.0).await
+	}
+
+	pub async fn handle_touch_up(&self, client: &mut Client, id: u32) -> Result<()> {
+		let serial = client.next_event_serial();
+		self.up(client, self.0, serial, 0, id as i32).await?;
+		self.frame(client, self.0).await
+	}
+
+	pub async fn reset(&self, client: &mut Client) -> Result<()> {
+		self.frame(client, self.0).await
+	}
+}
+
+impl WlTouch for Touch {
+	/// https://wayland.app/protocols/wayland#wl_touch:request:release
+	async fn release(
+		&self,
+		_client: &mut waynest::server::Client,
+		_sender_id: waynest::wire::ObjectId,
+	) -> Result<()> {
+		Ok(())
+	}
+}

src/wayland/data_device.rs

@@ -1,100 +0,0 @@
-use smithay::reexports::wayland_server::{
-	protocol::{
-		wl_data_device::{
-			Request::{Release, SetSelection, StartDrag},
-			WlDataDevice,
-		},
-		wl_data_device_manager::{
-			Request::{CreateDataSource, GetDataDevice},
-			WlDataDeviceManager,
-		},
-		wl_data_source::{
-			Request::{Destroy, Offer, SetActions},
-			WlDataSource,
-		},
-	},
-	Client, DataInit, Dispatch, DisplayHandle, GlobalDispatch, New, Resource,
-};
-
-use super::state::WaylandState;
-
-impl GlobalDispatch<WlDataDeviceManager, (), WaylandState> for WaylandState {
-	fn bind(
-		_state: &mut WaylandState,
-		_handle: &DisplayHandle,
-		_client: &Client,
-		resource: New<WlDataDeviceManager>,
-		_global_data: &(),
-		data_init: &mut DataInit<'_, WaylandState>,
-	) {
-		let _resource = data_init.init(resource, ());
-	}
-}
-
-impl Dispatch<WlDataDeviceManager, (), WaylandState> for WaylandState {
-	fn request(
-		_state: &mut WaylandState,
-		_client: &Client,
-		_resource: &WlDataDeviceManager,
-		request: <WlDataDeviceManager as Resource>::Request,
-		_data: &(),
-		_dhandle: &DisplayHandle,
-		data_init: &mut DataInit<'_, WaylandState>,
-	) {
-		match request {
-			CreateDataSource { id } => {
-				data_init.init(id, ());
-			}
-			GetDataDevice { id, seat: _ } => {
-				data_init.init(id, ());
-			}
-			_ => unreachable!(),
-		}
-	}
-}
-
-impl Dispatch<WlDataSource, (), WaylandState> for WaylandState {
-	fn request(
-		_state: &mut WaylandState,
-		_client: &Client,
-		_resource: &WlDataSource,
-		request: <WlDataSource as Resource>::Request,
-		_data: &(),
-		_dhandle: &DisplayHandle,
-		_data_init: &mut DataInit<'_, WaylandState>,
-	) {
-		match request {
-			Offer { mime_type: _ } => {}
-			Destroy => {}
-			SetActions { dnd_actions: _ } => {}
-			_ => unreachable!(),
-		}
-	}
-}
-
-impl Dispatch<WlDataDevice, (), WaylandState> for WaylandState {
-	fn request(
-		_state: &mut WaylandState,
-		_client: &Client,
-		_resource: &WlDataDevice,
-		request: <WlDataDevice as Resource>::Request,
-		_data: &(),
-		_dhandle: &DisplayHandle,
-		_data_init: &mut DataInit<'_, WaylandState>,
-	) {
-		match request {
-			StartDrag {
-				source: _,
-				origin: _,
-				icon: _,
-				serial: _,
-			} => {}
-			SetSelection {
-				source: _,
-				serial: _,
-			} => {}
-			Release => {}
-			_ => unreachable!(),
-		}
-	}
-}

src/wayland/decoration.rs

@@ -1,34 +0,0 @@
-use super::state::WaylandState;
-use smithay::{
-	delegate_kde_decoration, delegate_xdg_decoration,
-	reexports::wayland_protocols::xdg::decoration::zv1::server::zxdg_toplevel_decoration_v1::Mode,
-	wayland::shell::{
-		self, kde::decoration::KdeDecorationHandler, xdg::decoration::XdgDecorationHandler,
-	},
-};
-
-impl XdgDecorationHandler for WaylandState {
-	fn new_decoration(&mut self, toplevel: smithay::wayland::shell::xdg::ToplevelSurface) {
-		toplevel.with_pending_state(|state| {
-			state.decoration_mode = Some(Mode::ServerSide);
-		});
-		toplevel.send_configure();
-	}
-
-	fn request_mode(
-		&mut self,
-		_toplevel: smithay::wayland::shell::xdg::ToplevelSurface,
-		_mode: smithay::reexports::wayland_protocols::xdg::decoration::zv1::server::zxdg_toplevel_decoration_v1::Mode,
-	) {
-	}
-
-	fn unset_mode(&mut self, _toplevel: smithay::wayland::shell::xdg::ToplevelSurface) {}
-}
-delegate_xdg_decoration!(WaylandState);
-
-impl KdeDecorationHandler for WaylandState {
-	fn kde_decoration_state(&self) -> &shell::kde::decoration::KdeDecorationState {
-		&self.kde_decoration_state
-	}
-}
-delegate_kde_decoration!(WaylandState);

src/wayland/display.rs

@@ -0,0 +1,78 @@
+#![allow(unused)]
+
+use crate::wayland::{
+	MessageSink,
+	core::{
+		callback::{Callback, WlCallback},
+		output::Output,
+		seat::Seat,
+	},
+	registry::Registry,
+};
+use global_counter::primitive::exact::CounterU32;
+use std::{
+	sync::{Arc, OnceLock},
+	time::Instant,
+};
+pub use waynest::server::protocol::core::wayland::wl_display::*;
+use waynest::{
+	server::{Client, Dispatcher, Result},
+	wire::ObjectId,
+};
+
+#[derive(Dispatcher)]
+pub struct Display {
+	pub message_sink: MessageSink,
+	pub pid: Option<i32>,
+	pub seat: OnceLock<Arc<Seat>>,
+	pub output: OnceLock<Arc<Output>>,
+	id_counter: CounterU32,
+	pub creation_time: Instant,
+}
+impl Display {
+	pub fn new(message_sink: MessageSink, pid: Option<i32>) -> Self {
+		Self {
+			message_sink,
+			pid,
+			seat: OnceLock::new(),
+			output: OnceLock::new(),
+			id_counter: CounterU32::new(0xff000000), // Start at 0xff000000 to avoid conflicts with client-generated IDs
+			creation_time: Instant::now(),
+		}
+	}
+	pub fn next_server_id(&self) -> ObjectId {
+		unsafe { ObjectId::from_raw(self.id_counter.inc()) }
+	}
+}
+impl WlDisplay for Display {
+	/// https://wayland.app/protocols/wayland#wl_display:request:sync
+	async fn sync(
+		&self,
+		client: &mut Client,
+		sender_id: ObjectId,
+		callback_id: ObjectId,
+	) -> Result<()> {
+		let serial = client.next_event_serial();
+		Callback(callback_id)
+			.done(client, callback_id, serial)
+			.await?;
+
+		self.delete_id(client, sender_id, callback_id.as_raw())
+			.await?;
+		Ok(())
+	}
+
+	/// https://wayland.app/protocols/wayland#wl_display:request:get_registry
+	async fn get_registry(
+		&self,
+		client: &mut Client,
+		_sender_id: ObjectId,
+		registry_id: ObjectId,
+	) -> Result<()> {
+		let registry = client.insert(registry_id, Registry);
+
+		registry.advertise_globals(client, registry_id).await?;
+
+		Ok(())
+	}
+}

src/wayland/dmabuf/buffer_backing.rs

@@ -0,0 +1,120 @@
+use super::buffer_params::BufferParams;
+use crate::wayland::RENDER_DEVICE;
+use bevy::{
+	asset::{Assets, Handle},
+	image::Image,
+};
+use bevy_dmabuf::{
+	dmatex::{Dmatex, Resolution},
+	import::{DropCallback, ImportError, ImportedDmatexs, ImportedTexture, import_texture},
+};
+use drm_fourcc::DrmFourcc;
+use mint::Vector2;
+use parking_lot::Mutex;
+use std::sync::{Arc, OnceLock};
+use tracing::info;
+use waynest::server::protocol::stable::linux_dmabuf_v1::zwp_linux_buffer_params_v1::Flags;
+
+/// Parameters for a shared memory buffer
+pub struct DmabufBacking {
+	size: Vector2<u32>,
+	format: DrmFourcc,
+	tex: OnceLock<Handle<Image>>,
+	pending_imported_dmatex: Mutex<Option<ImportedTexture>>,
+}
+
+impl std::fmt::Debug for DmabufBacking {
+	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+		f.debug_struct("DmabufBacking")
+			.field("size", &self.size)
+			.field("format", &self.format)
+			.field("tex", &self.tex)
+			.finish()
+	}
+}
+
+impl DmabufBacking {
+	#[tracing::instrument(level = "debug", skip_all)]
+	pub fn new(dmatex: Dmatex) -> Result<Self, ImportError> {
+		let dev = RENDER_DEVICE.wait();
+
+		Ok(Self {
+			size: [dmatex.res.x, dmatex.res.y].into(),
+			format: DrmFourcc::try_from(dmatex.format).unwrap(),
+			tex: OnceLock::new(),
+			pending_imported_dmatex: Mutex::new(Some(import_texture(
+				dev,
+				dmatex,
+				DropCallback(None),
+			)?)),
+		})
+	}
+
+	#[tracing::instrument(level = "debug", skip_all)]
+	pub fn from_params(
+		params: Arc<BufferParams>,
+		size: Vector2<u32>,
+		format: DrmFourcc,
+		flags: Flags,
+	) -> Result<Self, ImportError> {
+		tracing::info!("Creating new DmabufBacking");
+		let mut planes = Vec::from_iter(std::mem::take(&mut *params.planes.lock()));
+		planes.sort_by_key(|(index, _)| *index);
+		let planes = planes.into_iter().map(|(_, tex)| tex).collect::<Vec<_>>();
+		let dmatex = Dmatex {
+			planes,
+			res: Resolution {
+				x: size.x,
+				y: size.y,
+			},
+			format: format as u32,
+			// TODO: impl this in bevy-dmabuf
+			flip_y: flags.contains(Flags::YInvert),
+			srgb: true,
+		};
+
+		DmabufBacking::new(dmatex)
+	}
+
+	#[tracing::instrument(level = "debug", skip_all)]
+	pub fn update_tex(
+		&self,
+		dmatexes: &ImportedDmatexs,
+		images: &mut Assets<Image>,
+	) -> Option<Handle<Image>> {
+		info!("updating dmabuf tex");
+		self.pending_imported_dmatex
+			.lock()
+			.take()
+			.map(|tex| dmatexes.insert_imported_dmatex(images, tex))
+			.inspect(|handle| {
+				_ = self.tex.set(handle.clone());
+			})
+	}
+
+	pub fn is_transparent(&self) -> bool {
+		matches!(
+			self.format,
+			DrmFourcc::Abgr1555
+				| DrmFourcc::Abgr16161616f
+				| DrmFourcc::Abgr2101010
+				| DrmFourcc::Abgr4444
+				| DrmFourcc::Abgr8888
+				| DrmFourcc::Argb1555
+				| DrmFourcc::Argb16161616f
+				| DrmFourcc::Argb2101010
+				| DrmFourcc::Argb4444
+				| DrmFourcc::Argb8888
+				| DrmFourcc::Axbxgxrx106106106106
+				| DrmFourcc::Ayuv
+				| DrmFourcc::Rgba1010102
+				| DrmFourcc::Rgba4444
+				| DrmFourcc::Rgba5551
+				| DrmFourcc::Rgba8888
+		)
+	}
+
+	pub fn size(&self) -> Vector2<usize> {
+		[self.size.x as usize, self.size.y as usize].into()
+	}
+}

src/wayland/dmabuf/buffer_params.rs

@@ -0,0 +1,180 @@
+use super::buffer_backing::DmabufBacking;
+use crate::wayland::{
+	core::buffer::{Buffer, BufferBacking},
+	util::ClientExt,
+};
+use bevy_dmabuf::dmatex::DmatexPlane;
+use drm_fourcc::DrmFourcc;
+use parking_lot::Mutex;
+use rustc_hash::FxHashMap;
+use std::os::fd::{AsRawFd, OwnedFd};
+use waynest::{
+	server::{
+		Client, Dispatcher, Result,
+		protocol::stable::linux_dmabuf_v1::zwp_linux_buffer_params_v1::{
+			Error, Flags, ZwpLinuxBufferParamsV1,
+		},
+	},
+	wire::ObjectId,
+};
+
+/// Parameters for creating a DMA-BUF-based wl_buffer
+///
+/// This is a temporary object that collects dmabufs and other parameters
+/// that together form a single logical buffer. The object may eventually
+/// create one wl_buffer unless cancelled by destroying it.
+#[derive(Debug, Dispatcher)]
+pub struct BufferParams {
+	pub id: ObjectId,
+	pub(super) planes: Mutex<FxHashMap<u32, DmatexPlane>>,
+}
+
+impl BufferParams {
+	#[tracing::instrument(level = "debug", skip_all)]
+	pub fn new(id: ObjectId) -> Self {
+		tracing::info!("Creating new BufferParams with id {:?}", id);
+		Self {
+			id,
+			planes: Mutex::new(FxHashMap::default()),
+		}
+	}
+}
+
+impl ZwpLinuxBufferParamsV1 for BufferParams {
+	async fn destroy(&self, _client: &mut Client, _sender_id: ObjectId) -> Result<()> {
+		tracing::info!("Destroying BufferParams {:?}", self.id);
+		Ok(())
+	}
+
+	#[tracing::instrument(level = "debug", skip_all)]
+	async fn add(
+		&self,
+		_client: &mut Client,
+		_sender_id: ObjectId,
+		fd: OwnedFd,
+		plane_idx: u32,
+		offset: u32,
+		stride: u32,
+		modifier_hi: u32,
+		modifier_lo: u32,
+	) -> Result<()> {
+		let fd_num = fd.as_raw_fd();
+		tracing::info!(
+			"Adding plane {} with fd {} to BufferParams {:?}",
+			plane_idx,
+			fd_num,
+			self.id
+		);
+
+		let mut planes = self.planes.lock();
+
+		// Check if plane index is already set
+		if planes.contains_key(&plane_idx) {
+			tracing::error!(
+				"Plane {} already exists in BufferParams {:?}",
+				plane_idx,
+				self.id
+			);
+			return Err(waynest::server::Error::MissingObject(self.id));
+		}
+
+		// Create plane with the provided parameters
+		let plane = DmatexPlane {
+			dmabuf_fd: fd.into(),
+			offset,
+			stride: stride as i32,
+			modifier: ((modifier_hi as u64) << 32) | (modifier_lo as u64),
+		};
+
+		// Store the plane
+		planes.insert(plane_idx, plane);
+		Ok(())
+	}
+
+	#[tracing::instrument(level = "debug", skip_all)]
+	async fn create(
+		&self,
+		client: &mut Client,
+		_sender_id: ObjectId,
+		width: i32,
+		height: i32,
+		format: u32,
+		flags: Flags,
+	) -> Result<()> {
+		tracing::info!("Creating buffer from BufferParams {:?}", self.id);
+		// Create the buffer with DMA-BUF backing using self as the backing
+		let size = [width as u32, height as u32].into();
+		let buffer = DmabufBacking::from_params(
+			client.get::<Self>(self.id).unwrap(),
+			size,
+			DrmFourcc::try_from(format).unwrap(),
+			flags,
+		)
+		.inspect_err(|e| tracing::error!("Failed to import dmabuf because {e}"))
+		.map(|backing| {
+			let id = client.display().next_server_id();
+			Buffer::new(client, id, BufferBacking::Dmabuf(backing))
+		});
+
+		match buffer {
+			Ok(buffer) => self.created(client, self.id, buffer.id).await,
+			Err(_) => {
+				client.remove(self.id);
+				self.failed(client, self.id).await
+			}
+		}
+	}
+
+	#[tracing::instrument(level = "debug", skip_all)]
+	async fn create_immed(
+		&self,
+		client: &mut Client,
+		sender_id: ObjectId,
+		buffer_id: ObjectId,
+		width: i32,
+		height: i32,
+		format: u32,
+		flags: Flags,
+	) -> Result<()> {
+		// TODO: terminate client on fail, or send a fail event or something
+		// Create the buffer with DMA-BUF backing using self as the backing
+		match DmabufBacking::from_params(
+			client.get::<Self>(self.id).unwrap(),
+			[width as u32, height as u32].into(),
+			DrmFourcc::try_from(format).unwrap(),
+			flags,
+		) {
+			Ok(backing) => {
+				Buffer::new(client, buffer_id, BufferBacking::Dmabuf(backing));
+			}
+			Err(e) => {
+				client
+					.protocol_error(
+						sender_id,
+						buffer_id,
+						Error::Incomplete as u32,
+						format!("Failed to import dmabuf because {e}"),
+					)
+					.await?;
+				tracing::error!("Failed to import dmabuf because {e}");
+			}
+		}
+		Ok(())
+	}
+}
+
+impl Drop for BufferParams {
+	#[tracing::instrument(level = "debug", skip_all)]
+	fn drop(&mut self) {
+		let planes = self.planes.get_mut();
+		tracing::info!("BufferParams being dropped with {} planes", planes.len());
+		for (idx, plane) in planes.iter() {
+			tracing::info!(
+				"Dropping plane {} with fd {}",
+				idx,
+				plane.dmabuf_fd.as_raw_fd()
+			);
+		}
+		planes.clear();
+	}
+}