Cargo.toml

@@ -21,37 +21,31 @@ name = "stardust-xr-server"
 path = "src/main.rs"
 
 [features]
-default = ["wayland"]
+default = []
 wayland = ["dep:smithay", "dep:wayland-scanner", "dep:wayland-backend"]
 profile_tokio = ["dep:console-subscriber", "tokio/tracing"]
-profile_app = ["dep:tracing-tracy"]
-local_deps = ["stereokit-rust/force-local-deps"]
+profile_app = ["dep:tracing-tracy", "bevy/trace_tracy", "bevy/trace"]
+bevy_debugging = ["bevy/bevy_remote", "bevy/track_location"]
 
 [package.metadata.appimage]
 auto_link = true
-auto_link_exclude_list = [
-	"libc*",
-	"libdl*",
-	"libpthread*",
-	"ld-linux*",
-	"libGL*",
-	"libEGL*",
-]
+auto_link_exclude_list = ["libc*", "libdl*", "libpthread*", "ld-linux*"]
 
 [profile.dev.package."*"]
 opt-level = 3
-debug = true
-strip = false
-debug-assertions = true
-overflow-checks = true
 
-[profile.release]
-opt-level = 3
-debug = "line-tables-only"
-strip = true
-debug-assertions = true
-overflow-checks = false
-lto = "thin"
+# [profile.release]
+# opt-level = 3
+# debug = "line-tables-only"
+# strip = true
+# debug-assertions = true
+# overflow-checks = false
+# lto = "thin"
+
+[patch.crates-io]
+bevy_mod_openxr = { git = "https://github.com/Schmarni-Dev/bevy_openxr", branch = "non_default_wait_frame_system" }
+bevy_mod_xr = { git = "https://github.com/Schmarni-Dev/bevy_openxr", branch = "non_default_wait_frame_system" }
+bevy_gltf = { git = "https://github.com/Schmarni-Dev/bevy", branch = "gltf_backport" }
 
 [dependencies]
 # small utility thingys
@@ -63,13 +57,14 @@ send_wrapper = "0.6.0"
 slotmap = "1.0.7"
 global_counter = "=0.2.2"
 parking_lot = "0.12.3"
+dashmap = "6.1.0"
 
 # rust errors/logging
 color-eyre = { version = "0.6.3", default-features = false }
 clap = { version = "4.5.13", features = ["derive"] }
 console-subscriber = { version = "0.4.0", optional = true }
 thiserror = "2.0.9"
-tracing = "0.1.40"
+tracing = { version = "0.1.40", features = ["release_max_level_warn"] }
 tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
 tracing-tracy = { version = "0.11.1", optional = true }
 
@@ -83,6 +78,17 @@ glam = { version = "0.29.0", features = ["mint", "serde"] }
 mint = "0.5.9"
 tokio = { version = "1.39.2", features = ["rt-multi-thread", "signal", "time"] }
 
+# bevy
+bevy = { version = "0.16", features = ["wayland", "mp3", "wav"] }
+bevy_mod_xr = "0.3"
+bevy_mod_openxr = "0.3"
+# bevy_sk.git = "https://github.com/MalekiRe/bevy_sk"
+bevy_sk = { git = "https://github.com/Schmarni-Dev/bevy_sk", branch = "fix_mat_stuff" }
+# bevy_sk.path = "../bevy_sk"
+bevy-mesh-text-3d.git = "https://github.com/terhechte/bevy-mesh-text-3d"
+
+openxr = "0.19"
+
 # linux stuffs
 input-event-codes = "6.2.0"
 zbus = { version = "5.0.0", default-features = false, features = ["tokio"] }
@@ -92,7 +98,8 @@ xkbcommon-rs = "0.1.0"
 # wayland
 wayland-backend = { version = "0.3.7", optional = true, default-features = false }
 wayland-scanner = { version = "0.31.4", optional = true }
-dashmap = "6.1.0"
+cosmic-text = "0.14.2"
+
 
 [dependencies.smithay]
 git = "https://github.com/smithay/smithay.git"
@@ -100,13 +107,6 @@ default-features = false
 features = ["desktop", "backend_drm", "renderer_gl", "wayland_frontend"]
 optional = true
 
-
-[dependencies.stereokit-rust]
-git = "https://github.com/mvvvv/StereoKit-rust.git"
-rev = "73ffaae6f42aa369e599a6ea0391f77840d682d8"
-features = ["no-event-loop"]
-default-features = false
-
 [dependencies.stardust-xr]
 workspace = true
 

src/core/bevy_channel.rs

@@ -0,0 +1,32 @@
+use std::sync::OnceLock;
+
+use bevy::prelude::*;
+use tokio::sync::mpsc::{self, error::TryRecvError};
+
+#[derive(Resource)]
+pub struct BevyChannelReader<T: Send + Sync + 'static>(mpsc::UnboundedReceiver<T>);
+pub struct BevyChannel<T: Send + Sync + 'static>(OnceLock<mpsc::UnboundedSender<T>>);
+impl<T: Send + Sync + 'static> BevyChannel<T> {
+	pub const fn new() -> Self {
+		Self(OnceLock::new())
+	}
+	pub fn init(&self, app: &mut App) {
+		let (tx, rx) = mpsc::unbounded_channel();
+		self.0.set(tx).unwrap();
+		app.insert_resource(BevyChannelReader(rx));
+	}
+	pub fn send(&self, msg: T) -> Option<()> {
+		self.0.get()?.send(msg).ok()
+	}
+}
+
+impl<T: Send + Sync + 'static> BevyChannelReader<T> {
+	pub fn read(&mut self) -> Option<T> {
+		match self.0.try_recv() {
+			Ok(v) => Some(v),
+			Err(TryRecvError::Disconnected) => panic!("bevy channel should never disconnect"),
+			Err(TryRecvError::Empty) => None,
+		}
+	}
+}
+

src/core/client.rs

@@ -86,7 +86,7 @@ impl Client {
 	pub fn from_connection(connection: UnixStream) -> Result<Arc<Self>> {
 		let pid = connection.peer_cred().ok().and_then(|c| c.pid());
 		let env = pid.and_then(|pid| get_env(pid).ok());
-		let exe = pid.and_then(|pid| fs::read_link(format!("/proc/{}/exe", pid)).ok());
+		let exe = pid.and_then(|pid| fs::read_link(format!("/proc/{pid}/exe")).ok());
 		info!(
 			pid,
 			exe = exe
@@ -201,7 +201,9 @@ impl Client {
 		std::fs::read_link(cwd_proc_path).ok()
 	}
 	pub async fn save_state(&self) -> Option<ClientStateParsed> {
+		println!("start save state");
 		let internal = self.root.get()?.save_state().await.ok()?;
+		println!("finished save state");
 		Some(ClientStateParsed::from_deserialized(self, internal))
 	}
 

src/core/color.rs

@@ -0,0 +1,12 @@
+use stardust_xr::values::color::{AlphaColor, Rgb, color_space::LinearRgb};
+
+pub trait ColorConvert {
+	fn to_bevy(&self) -> bevy::color::Color;
+}
+// even tho its supposed to be linear the values have to be interpreted as Srgba to produce the
+// correct result because StereoKit used Srgba while it was assumed that is uses linear rgba
+impl ColorConvert for AlphaColor<f32, Rgb<f32, LinearRgb>> {
+	fn to_bevy(&self) -> bevy::color::Color {
+		bevy::color::Color::srgba(self.c.r, self.c.g, self.c.b, self.a)
+	}
+}

src/core/entity_handle.rs

@@ -0,0 +1,33 @@
+use bevy::prelude::*;
+
+use super::bevy_channel::{BevyChannel, BevyChannelReader};
+pub struct EntityHandlePlugin;
+
+impl Plugin for EntityHandlePlugin {
+	fn build(&self, app: &mut App) {
+		DESTROY.init(app);
+		app.add_systems(PreUpdate, despawn);
+	}
+}
+
+fn despawn(mut cmds: Commands, mut reader: ResMut<BevyChannelReader<Entity>>) {
+	while let Some(e) = reader.read() {
+		cmds.entity(e).despawn();
+	}
+}
+
+static DESTROY: BevyChannel<Entity> = BevyChannel::new();
+#[derive(Deref, DerefMut, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)]
+pub struct EntityHandle(pub Entity);
+impl Drop for EntityHandle {
+	fn drop(&mut self) {
+		if DESTROY.send(self.0).is_none() {
+			error!("Entity Destroy channel not open");
+		}
+	}
+}
+impl From<Entity> for EntityHandle {
+	fn from(value: Entity) -> Self {
+		Self(value)
+	}
+}

src/core/error.rs

@@ -8,7 +8,6 @@ use stardust_xr::{
 		flexbuffers::{DeserializationError, ReaderError},
 	},
 };
-use stereokit_rust::StereoKitError;
 use thiserror::Error;
 
 pub type Result<T, E = ServerError> = std::result::Result<T, E>;
@@ -29,8 +28,6 @@ pub enum ServerError {
 	DeserializationError(#[from] DeserializationError),
 	#[error("Reader error: {0}")]
 	ReaderError(#[from] ReaderError),
-	#[error("StereoKit error: {0}")]
-	StereoKitError(#[from] StereoKitError),
 	#[error("Aspect {} does not exist for node", 0.to_string())]
 	NoAspect(TypeId),
 	#[error("{0}")]

src/core/mod.rs

@@ -7,3 +7,6 @@ pub mod registry;
 pub mod resource;
 pub mod scenegraph;
 pub mod task;
+pub mod color;
+pub mod entity_handle;
+pub mod bevy_channel;

src/main.rs

@@ -7,37 +7,69 @@ mod session;
 mod wayland;
 
 use crate::core::destroy_queue;
-use crate::nodes::items::camera;
-use crate::nodes::{audio, drawable, input};
+use crate::nodes::input;
 
+use bevy::MinimalPlugins;
+use bevy::a11y::AccessibilityPlugin;
+use bevy::app::{App, ScheduleRunnerPlugin, TerminalCtrlCHandlerPlugin};
+use bevy::asset::{AssetMetaCheck, UnapprovedPathMode};
+use bevy::audio::AudioPlugin;
+use bevy::core_pipeline::CorePipelinePlugin;
+use bevy::diagnostic::DiagnosticsPlugin;
+use bevy::ecs::schedule::{ExecutorKind, ScheduleLabel};
+use bevy::gizmos::GizmoPlugin;
+use bevy::gltf::GltfPlugin;
+use bevy::input::InputPlugin;
+use bevy::pbr::PbrPlugin;
+use bevy::render::{RenderDebugFlags, RenderPlugin};
+use bevy::scene::ScenePlugin;
+use bevy::text::FontLoader;
+use bevy::winit::{WakeUp, WinitPlugin};
+use bevy_mod_openxr::action_set_attaching::OxrActionAttachingPlugin;
+use bevy_mod_openxr::action_set_syncing::OxrActionSyncingPlugin;
+use bevy_mod_openxr::add_xr_plugins;
+use bevy_mod_openxr::exts::OxrExtensions;
+use bevy_mod_openxr::features::overlay::OxrOverlaySettings;
+use bevy_mod_openxr::features::passthrough::OxrPassthroughPlugin;
+use bevy_mod_openxr::init::{OxrInitPlugin, should_run_frame_loop};
+use bevy_mod_openxr::reference_space::OxrReferenceSpacePlugin;
+use bevy_mod_openxr::render::{OxrRenderPlugin, OxrWaitFrameSystem};
+use bevy_mod_openxr::resources::{OxrFrameState, OxrFrameWaiter, OxrSessionConfig};
+use bevy_mod_openxr::types::AppInfo;
+use bevy_mod_xr::camera::XrProjection;
+use bevy_mod_xr::session::{XrFirst, XrHandleEvents};
 use clap::Parser;
 use core::client::{Client, tick_internal_client};
+use core::entity_handle::EntityHandlePlugin;
 use core::task;
 use directories::ProjectDirs;
-use objects::ServerObjects;
+use nodes::audio::AudioNodePlugin;
+use nodes::drawable::lines::LinesNodePlugin;
+use nodes::drawable::model::ModelNodePlugin;
+use nodes::drawable::text::TextNodePlugin;
+use nodes::spatial::SpatialNodePlugin;
+use objects::input::mouse_pointer::FlatscreenInputPlugin;
+use objects::input::sk_controller::ControllerPlugin;
+use objects::input::sk_hand::HandPlugin;
+use objects::play_space::PlaySpacePlugin;
+use openxr::{EnvironmentBlendMode, ReferenceSpaceType};
 use session::{launch_start, save_session};
 use stardust_xr::schemas::dbus::object_registry::ObjectRegistry;
 use stardust_xr::server;
+use std::ops::DerefMut as _;
 use std::path::PathBuf;
 use std::sync::{Arc, OnceLock};
-use std::time::Duration;
-use stereokit_rust::material::Material;
-use stereokit_rust::shader::Shader;
-use stereokit_rust::sk::{
-	AppMode, DepthMode, DisplayBlend, OriginMode, QuitReason, SkSettings, sk_quit,
-};
-use stereokit_rust::system::{Handed, Input, LogLevel, Renderer};
-use stereokit_rust::tex::{SHCubemap, Tex, TexFormat, TexType};
-use stereokit_rust::ui::Ui;
-use stereokit_rust::util::{Color128, SphericalHarmonics, Time};
 use tokio::net::UnixListener;
 use tokio::sync::Notify;
+use tokio::task::JoinError;
 use tracing::metadata::LevelFilter;
-use tracing::{debug_span, error, info};
+use tracing::{error, info};
 use tracing_subscriber::{EnvFilter, fmt, prelude::*};
 use zbus::Connection;
 use zbus::fdo::ObjectManager;
 
+use bevy::prelude::*;
+
 #[derive(Debug, Clone, Parser)]
 #[clap(author, version, about, long_about = None)]
 struct CliArgs {
@@ -73,11 +105,13 @@ struct CliArgs {
 	nvidia: bool,
 }
 
+pub type BevyMaterial = StandardMaterial;
+
 static STARDUST_INSTANCE: OnceLock<String> = OnceLock::new();
 
 // #[tokio::main(flavor = "current_thread")]
 #[tokio::main]
-async fn main() {
+async fn main() -> Result<AppExit, JoinError> {
 	color_eyre::install().unwrap();
 
 	let registry = tracing_subscriber::registry();
@@ -169,15 +203,15 @@ async fn main() {
 		"Couldn't make the object registry to find all objects with given interfaces in d-bus",
 	);
 
-	let sk_ready_notifier = Arc::new(Notify::new());
-	let stereokit_loop = tokio::task::spawn_blocking({
-		let sk_ready_notifier = sk_ready_notifier.clone();
+	let ready_notifier = Arc::new(Notify::new());
+	let io_loop = tokio::task::spawn_blocking({
+		let ready_notifier = ready_notifier.clone();
 		let project_dirs = project_dirs.clone();
 		let cli_args = cli_args.clone();
 		let dbus_connection = dbus_connection.clone();
 		move || {
-			stereokit_loop(
-				sk_ready_notifier,
+			bevy_loop(
+				ready_notifier,
 				project_dirs,
 				cli_args,
 				dbus_connection,
@@ -185,16 +219,12 @@ async fn main() {
 			)
 		}
 	});
-	sk_ready_notifier.notified().await;
+	ready_notifier.notified().await;
 	let mut startup_children = project_dirs
 		.as_ref()
 		.map(|project_dirs| launch_start(&cli_args, project_dirs))
 		.unwrap_or_default();
-
-	tokio::select! {
-		_ = stereokit_loop => (),
-		_ = tokio::signal::ctrl_c() => unsafe {sk_quit(QuitReason::SystemClose)},
-	}
+	let return_value = io_loop.await;
 	info!("Stopping...");
 	if let Some(project_dirs) = project_dirs {
 		save_session(&project_dirs).await;
@@ -204,141 +234,235 @@ async fn main() {
 	}
 
 	info!("Cleanly shut down Stardust");
+	return_value
 }
 
-static DEFAULT_SKYTEX: OnceLock<Tex> = OnceLock::new();
-static DEFAULT_SKYLIGHT: OnceLock<SphericalHarmonics> = OnceLock::new();
+// static DEFAULT_SKYTEX: OnceLock<Tex> = OnceLock::new();
+// static DEFAULT_SKYLIGHT: OnceLock<SphericalHarmonics> = OnceLock::new();
 
-fn stereokit_loop(
-	sk_ready_notifier: Arc<Notify>,
-	project_dirs: Option<ProjectDirs>,
+#[derive(ScheduleLabel, Hash, Debug, PartialEq, Eq, Clone, Copy)]
+pub struct PreFrameWait;
+#[derive(Resource, Deref)]
+pub struct ObjectRegistryRes(ObjectRegistry);
+#[derive(Resource, Deref)]
+pub struct DbusConnection(Connection);
+
+fn bevy_loop(
+	ready_notifier: Arc<Notify>,
+	_project_dirs: Option<ProjectDirs>,
 	args: CliArgs,
 	dbus_connection: Connection,
 	object_registry: ObjectRegistry,
-) {
-	let sk = SkSettings::default()
-		.app_name("Stardust XR")
-		.blend_preference(DisplayBlend::AnyTransparent)
-		.mode(if args.flatscreen {
-			AppMode::Simulator
-		} else {
-			AppMode::XR
+) -> AppExit {
+	let mut app = App::new();
+	app.insert_resource(DbusConnection(dbus_connection));
+	app.add_plugins(AssetPlugin {
+		meta_check: AssetMetaCheck::Never,
+		unapproved_path_mode: UnapprovedPathMode::Allow,
+		..default()
+	});
+	let mut plugins = MinimalPlugins
+		.build()
+		.add(DiagnosticsPlugin)
+		.add(TransformPlugin)
+		.add(InputPlugin)
+		.add(AccessibilityPlugin);
+	plugins = plugins
+		.add(TerminalCtrlCHandlerPlugin)
+		// bevy_mod_openxr will replace this, TODO: figure out how to mix this with
+		// bevy-dmabuf
+		.add(RenderPlugin::default())
+		.add(ImagePlugin::default())
+		.add(CorePipelinePlugin)
+		// theoretically we shouldn't need this because of bevy_sk, but everything is tangled in
+		// there and idk what we actually need to run
+		.add(PbrPlugin {
+			// this seems to only apply to StandardMaterial, we don't use that
+			prepass_enabled: true,
+			add_default_deferred_lighting_plugin: false,
+			use_gpu_instance_buffer_builder: true,
+			debug_flags: RenderDebugFlags::default(),
 		})
-		.depth_mode(DepthMode::D32)
-		.log_filter(match EnvFilter::from_default_env().max_level_hint() {
-			Some(LevelFilter::ERROR) => LogLevel::Error,
-			Some(LevelFilter::WARN) => LogLevel::Warning,
-			Some(LevelFilter::INFO) => LogLevel::Inform,
-			Some(LevelFilter::DEBUG) => LogLevel::Diagnostic,
-			Some(LevelFilter::TRACE) => LogLevel::Diagnostic,
-			Some(LevelFilter::OFF) => LogLevel::None,
-			None => LogLevel::Warning,
-		})
-		.overlay_app(args.overlay_priority.is_some())
-		.overlay_priority(args.overlay_priority.unwrap_or(u32::MAX))
-		.disable_desktop_input_window(true)
-		.origin(OriginMode::Local)
-		.init()
-		.expect("StereoKit failed to initialize");
-	info!("Init StereoKit");
-
-	Renderer::multisample(0);
-	Material::default().shader(Shader::pbr_clip());
-	Ui::enable_far_interact(false);
-
-	let left_hand_material = Material::find("default/material_hand").unwrap();
-	let mut right_hand_material = left_hand_material.copy();
-	right_hand_material.id("right_hand");
-	Input::hand_material(Handed::Right, Some(Material::find("right_hand").unwrap()));
-
-	Input::hand_visible(Handed::Left, false);
-	Input::hand_visible(Handed::Right, false);
-
-	// Skytex/light stuff
+		// required for gltf
+		.add(ScenePlugin)
+		.add(GltfPlugin::default())
+		.add(AudioPlugin::default())
+		.add(GizmoPlugin);
+	let mut task_pool_plugin = TaskPoolPlugin::default();
+	// make tokio work
+	let handle = tokio::runtime::Handle::current();
+	let enter_runtime_context = Arc::new(move || {
+		// TODO: this might be a memory leak
+		std::mem::forget(handle.enter());
+	});
+	task_pool_plugin.task_pool_options.io.on_thread_spawn = Some(enter_runtime_context.clone());
+	task_pool_plugin.task_pool_options.compute.on_thread_spawn =
+		Some(enter_runtime_context.clone());
+	task_pool_plugin
+		.task_pool_options
+		.async_compute
+		.on_thread_spawn = Some(enter_runtime_context.clone());
+	plugins = plugins.set(task_pool_plugin);
+	if args.flatscreen
+		|| std::env::var_os("DISPLAY").is_some()
+		|| std::env::var_os("WAYLAND_DISPLAY").is_some()
 	{
-		let _ = DEFAULT_SKYTEX.set(Tex::gen_color(
-			Color128::BLACK,
-			1,
-			1,
-			TexType::Cubemap,
-			TexFormat::RGBA32,
-		));
-		let _ = DEFAULT_SKYLIGHT.set(Renderer::get_skylight());
-		if let Some(sky) = project_dirs
-			.as_ref()
-			.map(|dirs| dirs.config_dir().join("skytex.hdr"))
-			.filter(|f| f.exists())
-			.and_then(|p| SHCubemap::from_cubemap(p, true, 100).ok())
-		{
-			sky.render_as_sky();
-		} else {
-			Renderer::skytex(DEFAULT_SKYTEX.get().unwrap());
-		}
+		let mut plugin = WinitPlugin::<WakeUp>::default();
+		plugin.run_on_any_thread = true;
+		plugins = plugins
+			.add(plugin)
+			.disable::<ScheduleRunnerPlugin>()
+			.add(FlatscreenInputPlugin);
 	}
-
-	#[cfg(feature = "wayland")]
-	let mut wayland = wayland::Wayland::new().expect("Could not initialize wayland");
-	#[cfg(feature = "wayland")]
-	wayland.make_context_current();
-	sk_ready_notifier.notify_waiters();
-	info!("Stardust ready!");
-
-	let mut objects = ServerObjects::new(
-		dbus_connection.clone(),
-		&sk,
-		[left_hand_material, right_hand_material],
-		args.disable_controllers,
-		args.disable_hands,
+	app.add_plugins(
+		add_xr_plugins(plugins.add(WindowPlugin::default()))
+			.set(OxrInitPlugin {
+				app_info: AppInfo {
+					name: "Stardust XR".into(),
+					version: bevy_mod_openxr::types::Version(0, 44, 1),
+				},
+				exts: {
+					// all OpenXR extensions can be requested here
+					let mut exts = OxrExtensions::default();
+					exts.enable_hand_tracking();
+					if args.overlay_priority.is_some() {
+						exts.enable_extx_overlay();
+					}
+					exts
+				},
+				..default()
+			})
+			.set(OxrRenderPlugin {
+				default_wait_frame: false,
+				..default()
+			})
+			.set(OxrReferenceSpacePlugin {
+				default_primary_ref_space: ReferenceSpaceType::LOCAL,
+			})
+			// Disable a bunch of unneeded plugins
+			// this plugin uses the fb extention, blend mode still works
+			.disable::<OxrPassthroughPlugin>()
+			// we don't do any action stuff that needs to integrate with the ecosystem
+			.disable::<OxrActionAttachingPlugin>()
+			.disable::<OxrActionSyncingPlugin>(),
 	);
 
-	let mut last_frame_delta = Duration::ZERO;
-	let mut sleep_duration = Duration::ZERO;
-	while let Some(token) = sk.step() {
-		let _span = debug_span!("StereoKit step");
-		let _span = _span.enter();
-
-		camera::update(token);
-		#[cfg(feature = "wayland")]
-		wayland.frame_event();
-		destroy_queue::clear();
-
-		objects.update(&sk, token, &dbus_connection, &object_registry);
-		input::process_input();
-		nodes::root::Root::send_frame_events(Time::get_step_unscaled());
-		adaptive_sleep(
-			&mut last_frame_delta,
-			&mut sleep_duration,
-			Duration::from_micros(250),
-		);
-
-		tick_internal_client();
-		#[cfg(feature = "wayland")]
-		wayland.update();
-		drawable::draw(token);
-		audio::update();
+	app.add_plugins((
+		bevy_sk::hand::HandPlugin,
+		// bevy_sk::vr_materials::SkMaterialPlugin {
+		// 	replace_standard_material: false,
+		// },
+		// bevy_sk::skytext::SphericalHarmonicsPlugin,
+	));
+	// app.add_plugins(HandGizmosPlugin);
+	app.init_asset::<Font>().init_asset_loader::<FontLoader>();
+	app.world_mut().resource_mut::<AmbientLight>().brightness = 2000.0;
+	if let Some(priority) = args.overlay_priority {
+		app.insert_resource(OxrOverlaySettings {
+			session_layer_placement: priority,
+			..default()
+		});
 	}
+	app.insert_resource(OxrSessionConfig {
+		blend_modes: Some(vec![
+			EnvironmentBlendMode::ALPHA_BLEND,
+			EnvironmentBlendMode::ADDITIVE,
+			EnvironmentBlendMode::OPAQUE,
+		]),
+		..default()
+	});
+	let mut pre_frame_wait = Schedule::new(PreFrameWait);
+	pre_frame_wait.set_executor_kind(ExecutorKind::MultiThreaded);
+	app.add_schedule(pre_frame_wait);
+	app.insert_resource(ClearColor(Color::BLACK.with_alpha(0.0)));
+	app.insert_resource(ObjectRegistryRes(object_registry));
+	#[cfg(feature = "bevy_debugging")]
+	{
+		app.add_plugins((RemotePlugin::default(), RemoteHttpPlugin::default()));
+	}
+	// the Stardust server plugins
+	// infra plugins
+	app.add_plugins(EntityHandlePlugin);
+	// node plugins
+	app.add_plugins((
+		SpatialNodePlugin,
+		ModelNodePlugin,
+		TextNodePlugin,
+		LinesNodePlugin,
+		AudioNodePlugin,
+	));
+	// object plugins
+	app.add_plugins((PlaySpacePlugin, HandPlugin, ControllerPlugin));
+	app.add_systems(PostStartup, move || {
+		ready_notifier.notify_waiters();
+	});
+	app.add_systems(Update, update_cameras);
+	app.add_systems(
+		XrFirst,
+		xr_step
+			.in_set(OxrWaitFrameSystem)
+			.in_set(XrHandleEvents::FrameLoop),
+	);
 
-	info!("Cleanly shut down StereoKit");
+	app.run()
 }
-
-fn adaptive_sleep(
-	last_frame_delta: &mut Duration,
-	sleep_duration: &mut Duration,
-	sleep_duration_increase: Duration,
-) {
-	let frame_delta = Duration::from_secs_f64(Time::get_step_unscaled());
-	if *last_frame_delta < frame_delta {
-		if let Some(frame_delta_delta) = frame_delta.checked_sub(*last_frame_delta) {
-			if let Some(new_sleep_duration) = sleep_duration.checked_sub(frame_delta_delta) {
-				*sleep_duration = new_sleep_duration;
+fn update_cameras(mut camera: Query<&mut Projection, (With<Camera3d>,)>) {
+	for mut projection in &mut camera {
+		match projection.deref_mut() {
+			Projection::Perspective(perspective_projection) => perspective_projection.near = 0.003,
+			Projection::Orthographic(orthographic_projection) => {
+				orthographic_projection.near = 0.003
+			}
+			Projection::Custom(custom_projection) => {
+				if let Some(xr) = custom_projection.get_mut::<XrProjection>() {
+					xr.near = 0.003
+				} else {
+					error_once!("unknown custom camera projection");
+				}
 			}
 		}
-	} else {
-		*sleep_duration += sleep_duration_increase;
+	}
+}
+
+fn xr_step(world: &mut World) {
+	// camera::update(token);
+	#[cfg(feature = "wayland")]
+	wayland.frame_event();
+	destroy_queue::clear();
+
+	// update things like the Xr input methods
+	world.run_schedule(PreFrameWait);
+	input::process_input();
+	let time = world.resource::<bevy::prelude::Time>().delta_secs_f64();
+	nodes::root::Root::send_frame_events(time);
+
+	// we are targeting the frame after the wait
+	if let Some(mut state) = world.get_resource_mut::<OxrFrameState>() {
+		state.predicted_display_time = openxr::Time::from_nanos(
+			state.predicted_display_time.as_nanos() + state.predicted_display_period.as_nanos(),
+		);
 	}
 
-	debug_span!("Sleep", ?sleep_duration, ?frame_delta, ?last_frame_delta).in_scope(|| {
-		*last_frame_delta = frame_delta;
-		std::thread::sleep(*sleep_duration); // to give clients a chance to even update anything before drawing
-	});
+	let should_wait = world
+		.run_system_cached(should_run_frame_loop)
+		.unwrap_or(false);
+	// we might want to do an adaptive sleep when not OpenXR waiting
+	if should_wait {
+		world.resource_scope::<OxrFrameWaiter, _>(|world, mut waiter| {
+			let state = waiter
+				.wait()
+				.inspect_err(|err| error!("failed to wait OpenXR frame: {err}"))
+				.ok();
+
+			if let Some(state) = state {
+				world.insert_resource(OxrFrameState(state));
+			}
+		});
+	}
+
+	tick_internal_client();
+	#[cfg(feature = "wayland")]
+	wayland.update();
+	// drawable::draw(token);
+	// audio::update();
 }

src/nodes/audio.rs

@@ -1,30 +1,104 @@
+use super::spatial::SpatialNode;
 use super::{Aspect, AspectIdentifier, Node};
 use crate::core::client::Client;
-use crate::core::destroy_queue;
+use crate::core::entity_handle::EntityHandle;
 use crate::core::error::Result;
 use crate::core::registry::Registry;
 use crate::core::resource::get_resource_file;
 use crate::nodes::spatial::{SPATIAL_ASPECT_ALIAS_INFO, Spatial, Transform};
+use bevy::audio::{PlaybackMode, Volume};
+use bevy_mod_openxr::session::OxrSession;
+use bevy_mod_xr::session::{XrPreDestroySession, XrSessionCreated};
+use bevy_mod_xr::spaces::XrSpace;
 use color_eyre::eyre::eyre;
-use glam::{Vec4Swizzles, vec3};
 use parking_lot::Mutex;
 use stardust_xr::values::ResourceID;
 
-use std::ops::DerefMut;
+use bevy::prelude::*;
+use bevy::transform::components::Transform as BevyTransform;
 use std::sync::{Arc, OnceLock};
 use std::{ffi::OsStr, path::PathBuf};
-use stereokit_rust::sound::{Sound as SkSound, SoundInst};
+
+pub struct AudioNodePlugin;
+impl Plugin for AudioNodePlugin {
+	fn build(&self, app: &mut App) {
+		app.add_systems(Update, update_sound_event);
+		app.add_systems(XrSessionCreated, spawn_hmd_audio_listener);
+		app.add_systems(XrPreDestroySession, despawn_hmd_audio_listener);
+	}
+}
+
+fn despawn_hmd_audio_listener(mut cmds: Commands, session: Res<OxrSession>, res: Res<HmdListener>) {
+	cmds.remove_resource::<HmdListener>();
+	cmds.entity(res.0).despawn();
+	_ = session.destroy_space(res.1);
+}
+
+fn spawn_hmd_audio_listener(mut cmds: Commands, session: Res<OxrSession>) {
+	let space = session
+		.create_reference_space(openxr::ReferenceSpaceType::VIEW, BevyTransform::IDENTITY)
+		.unwrap();
+	let listener = cmds
+		.spawn((
+			Name::new("HMD audio listener"),
+			space.0,
+			SpatialListener::new(0.2),
+		))
+		.id();
+	cmds.insert_resource(HmdListener(listener, space.0));
+}
+#[derive(Resource)]
+struct HmdListener(Entity, XrSpace);
+fn update_sound_event(
+	mut cmds: Commands,
+	sinks: Query<&SpatialAudioSink>,
+	asset_server: Res<AssetServer>,
+) {
+	for sound in SOUND_REGISTRY.get_valid_contents() {
+		if sound.entity.get().is_none() {
+			let handle = asset_server.load(sound.pending_audio_path.as_path());
+			sound
+				.entity
+				.set(
+					cmds.spawn((
+						Name::new("Audio Node"),
+						SpatialNode(Arc::downgrade(&sound.spatial)),
+						AudioPlayer::new(handle),
+						PlaybackSettings {
+							mode: PlaybackMode::Once,
+							volume: Volume::Linear(sound.volume),
+							speed: 1.0,
+							paused: true,
+							muted: false,
+							spatial: true,
+							spatial_scale: None,
+						},
+					))
+					.id()
+					.into(),
+				)
+				.unwrap();
+		}
+		if let Some(sink) = sound.entity.get().and_then(|e| sinks.get(e.0).ok()) {
+			if sound.play.lock().take().is_some() {
+				sink.play();
+			}
+			if sound.stop.lock().take().is_some() {
+				sink.stop();
+			}
+		}
+	}
+}
 
 static SOUND_REGISTRY: Registry<Sound> = Registry::new();
 
 stardust_xr_server_codegen::codegen_audio_protocol!();
 pub struct Sound {
-	space: Arc<Spatial>,
+	spatial: Arc<Spatial>,
 
 	volume: f32,
 	pending_audio_path: PathBuf,
-	sk_sound: OnceLock<SkSound>,
-	instance: Mutex<Option<SoundInst>>,
+	entity: OnceLock<EntityHandle>,
 	stop: Mutex<Option<()>>,
 	play: Mutex<Option<()>>,
 }
@@ -37,11 +111,10 @@ impl Sound {
 		)
 		.ok_or_else(|| eyre!("Resource not found"))?;
 		let sound = Sound {
-			space: node.get_aspect::<Spatial>().unwrap().clone(),
+			spatial: node.get_aspect::<Spatial>().unwrap().clone(),
 			volume: 1.0,
 			pending_audio_path,
-			sk_sound: OnceLock::new(),
-			instance: Mutex::new(None),
+			entity: OnceLock::new(),
 			stop: Mutex::new(None),
 			play: Mutex::new(None),
 		};
@@ -49,24 +122,6 @@ impl Sound {
 		node.add_aspect_raw(sound_arc.clone());
 		Ok(sound_arc)
 	}
-
-	fn update(&self) {
-		let sound = self
-			.sk_sound
-			.get_or_init(|| SkSound::from_file(self.pending_audio_path.clone()).unwrap());
-		if self.stop.lock().take().is_some() {
-			if let Some(instance) = self.instance.lock().take() {
-				instance.stop();
-			}
-		}
-		if self.instance.lock().is_none() && self.play.lock().take().is_some() {
-			let instance = sound.play(vec3(0.0, 0.0, 0.0), Some(self.volume));
-			self.instance.lock().replace(instance);
-		}
-		if let Some(instance) = self.instance.lock().deref_mut() {
-			instance.position(self.space.global_transform().w_axis.xyz());
-		}
-	}
 }
 impl AspectIdentifier for Sound {
 	impl_aspect_for_sound_aspect_id! {}
@@ -88,22 +143,10 @@ impl SoundAspect for Sound {
 }
 impl Drop for Sound {
 	fn drop(&mut self) {
-		if let Some(instance) = self.instance.lock().take() {
-			instance.stop();
-		}
-		if let Some(sk_sound) = self.sk_sound.take() {
-			destroy_queue::add(sk_sound);
-		}
 		SOUND_REGISTRY.remove(self);
 	}
 }
 
-pub fn update() {
-	for sound in SOUND_REGISTRY.get_valid_contents() {
-		sound.update()
-	}
-}
-
 impl InterfaceAspect for Interface {
 	#[doc = "Create a sound node. WAV and MP3 are supported."]
 	fn create_sound(

src/nodes/drawable/lines.rs

@@ -1,20 +1,226 @@
 use super::{Line, LinesAspect};
 use crate::{
-	core::{client::Client, error::Result, registry::Registry},
-	nodes::{Node, spatial::Spatial},
+	core::{
+		client::Client, color::ColorConvert, entity_handle::EntityHandle, error::Result,
+		registry::Registry,
+	},
+	nodes::{
+		spatial::{Spatial, SpatialNode}, Node
+	}, BevyMaterial,
 };
-use glam::{FloatExt, Vec3};
+use bevy::{
+	asset::RenderAssetUsages,
+	prelude::*,
+	render::{
+		mesh::{Indices, MeshAabb, PrimitiveTopology},
+		primitives::Aabb,
+	},
+};
+use glam::Vec3;
 use parking_lot::Mutex;
-use std::{collections::VecDeque, sync::Arc};
-use stereokit_rust::{
-	maths::Bounds, sk::MainThreadToken, system::LinePoint as SkLinePoint, util::Color128,
+use std::sync::{
+	Arc, OnceLock,
+	atomic::{AtomicBool, Ordering},
 };
 
+pub struct LinesNodePlugin;
+
+impl Plugin for LinesNodePlugin {
+	fn build(&self, app: &mut App) {
+		app.add_systems(Update, (build_line_mesh/* , update_visibillity */).chain());
+	}
+}
+
+fn update_visibillity(mut cmds: Commands) {
+	for lines in LINES_REGISTRY.get_valid_contents().into_iter() {
+		let Some(entity) = lines.entity.get().map(|e| **e) else {
+			continue;
+		};
+		match lines.spatial.node().map(|n| n.enabled()).unwrap_or(false) {
+			true => {
+				cmds.entity(entity)
+					.insert_recursive::<Children>(Visibility::Visible);
+			}
+			false => {
+				cmds.entity(entity)
+					.insert_recursive::<Children>(Visibility::Hidden);
+			}
+		}
+	}
+}
+
+fn build_line_mesh(
+	mut meshes: ResMut<Assets<Mesh>>,
+	mut cmds: Commands,
+	mut materials: ResMut<Assets<BevyMaterial>>,
+) {
+	for lines in LINES_REGISTRY
+		.get_valid_contents()
+		.into_iter()
+		.filter(|l| l.gen_mesh.load(Ordering::Relaxed))
+	{
+		lines.gen_mesh.store(false, Ordering::Relaxed);
+		let mut vertex_positions = Vec::<Vec3>::new();
+		let mut vertex_normals = Vec::<Vec3>::new();
+		let mut vertex_colors = Vec::<[f32; 4]>::new();
+		let mut vertex_indecies = Vec::<u32>::new();
+		let lines_data = lines.data.lock();
+		if lines_data.is_empty() {
+			*lines.bounds.lock() = Aabb::default();
+			match lines.entity.get() {
+				Some(e) => cmds.entity(**e),
+				None => {
+					let e = cmds.spawn((
+						Name::new("LinesNode"),
+						SpatialNode(Arc::downgrade(&lines.spatial)),
+					));
+					_ = lines.entity.set(e.id().into());
+					e
+				}
+			}
+			.remove::<Mesh3d>();
+			continue;
+		}
+
+		let mut indecies_set = 0;
+		for line in lines_data.iter() {
+			let start_set = indecies_set;
+			let optional_points = {
+				let mut out = Vec::new();
+				let mut last = line.cyclic.then(|| line.points.last()).flatten();
+				let mut peekable = line.points.iter().peekable();
+				while let Some(curr) = peekable.next() {
+					let mut end = false;
+					let next = match peekable.peek() {
+						Some(v) => Some(*v),
+						None => {
+							end = true;
+							line.cyclic.then(|| line.points.first()).flatten()
+						}
+					};
+
+					out.push((last, curr, next, end));
+					last = Some(curr);
+				}
+				out
+			};
+			for (last, curr, next, end) in optional_points {
+				let last_quat = last.map(|v| {
+					Quat::from_rotation_arc(
+						Vec3::Y,
+						(Vec3::from(curr.point) - Vec3::from(v.point)).normalize(),
+					)
+				});
+				let next_quat = next.map(|v| {
+					Quat::from_rotation_arc(
+						Vec3::Y,
+						(Vec3::from(v.point) - Vec3::from(curr.point)).normalize(),
+					)
+				});
+				let quat = match (last_quat, next_quat) {
+					(None, None) => unreachable!(),
+					(None, Some(next)) => next,
+					(Some(last), None) => last,
+					(Some(last), Some(next)) => last.lerp(next, 0.5),
+				};
+				let normals = [
+					Vec3::X,
+					Vec3::new(1., 0., 1.).normalize(),
+					Vec3::Z,
+					Vec3::new(-1., 0., 1.).normalize(),
+					Vec3::NEG_X,
+					Vec3::new(-1., 0., -1.).normalize(),
+					Vec3::NEG_Z,
+					Vec3::new(1., 0., -1.).normalize(),
+				]
+				.map(Vec3::normalize)
+				.map(|v| (quat * v));
+				let points = normals.map(|v| (v * curr.thickness) + Vec3::from(curr.point));
+				vertex_normals.extend(normals);
+				vertex_positions.extend(points);
+				vertex_colors.extend([curr.color.to_bevy().to_srgba().to_f32_array(); 8]);
+				if !end {
+					vertex_indecies.extend(indecies(indecies_set));
+				}
+				indecies_set += 1;
+			}
+			if line.cyclic {
+				vertex_indecies.extend(cyclic_indecies(start_set, indecies_set - 1));
+			} else {
+				vertex_indecies.extend(cap_indecies(start_set, false));
+				vertex_indecies.extend(cap_indecies(indecies_set - 1, true));
+			}
+		}
+		let mut mesh = Mesh::new(
+			PrimitiveTopology::TriangleList,
+			RenderAssetUsages::RENDER_WORLD,
+		);
+		mesh.insert_indices(Indices::U32(vertex_indecies));
+		mesh.insert_attribute(Mesh::ATTRIBUTE_COLOR, vertex_colors);
+		mesh.insert_attribute(Mesh::ATTRIBUTE_NORMAL, vertex_normals);
+		mesh.insert_attribute(Mesh::ATTRIBUTE_POSITION, vertex_positions.clone());
+		if let Some(aabb) = mesh.compute_aabb() {
+			*lines.bounds.lock() = aabb;
+		}
+
+		match lines.entity.get() {
+			Some(e) => cmds.entity(**e),
+			None => {
+				let e = cmds.spawn((
+					Name::new("LinesNode"),
+					SpatialNode(Arc::downgrade(&lines.spatial)),
+					MeshMaterial3d(materials.add(BevyMaterial {
+						base_color: Color::WHITE,
+						perceptual_roughness: 1.0,
+						alpha_mode: AlphaMode::Opaque,
+						..default()
+					})),
+				));
+				_ = lines.entity.set(e.id().into());
+				e
+			}
+		}
+		.insert(Mesh3d(meshes.add(mesh)));
+	}
+}
+
+const END_CAP_INDECIES: [u32; 18] = [0, 1, 7, 7, 1, 2, 7, 2, 6, 6, 2, 3, 6, 3, 5, 5, 3, 4];
+fn cap_indecies(set: u32, flip: bool) -> [u32; END_CAP_INDECIES.len()] {
+	let mut out = END_CAP_INDECIES.map(|v| v + (set * 8));
+	if flip {
+		out.reverse();
+	}
+	out
+}
+
+// const BASE: [u16; 6] = [0, 8, 1, 8, 9, 1];
+const INDECIES: [u32; 48] = [
+	0, 8, 1, 8, 9, 1, 1, 9, 2, 9, 10, 2, 2, 10, 3, 10, 11, 3, 3, 11, 4, 11, 12, 4, 4, 12, 5, 12,
+	13, 5, 5, 13, 6, 13, 14, 6, 6, 14, 7, 14, 15, 7, 7, 15, 0, 15, 8, 0,
+];
+fn indecies(set: u32) -> [u32; INDECIES.len()] {
+	INDECIES.map(|v| v + (set * 8))
+}
+fn cyclic_indecies(start_set: u32, end_set: u32) -> [u32; INDECIES.len()] {
+	let mut out = INDECIES.map(|v| {
+		if v < 8 {
+			v + ((start_set) * 8)
+		} else {
+			v + ((end_set - 1) * 8)
+		}
+	});
+	out.reverse();
+	out
+}
+
 static LINES_REGISTRY: Registry<Lines> = Registry::new();
 
 pub struct Lines {
-	space: Arc<Spatial>,
+	spatial: Arc<Spatial>,
 	data: Mutex<Vec<Line>>,
+	gen_mesh: AtomicBool,
+	entity: OnceLock<EntityHandle>,
+	bounds: Mutex<Aabb>,
 }
 impl Lines {
 	pub fn add_to(node: &Arc<Node>, lines: Vec<Line>) -> Result<Arc<Lines>> {
@@ -23,67 +229,29 @@ impl Lines {
 			.unwrap()
 			.bounding_box_calc
 			.set(|node| {
-				let mut bounds = Bounds::default();
-				if let Ok(lines) = node.get_aspect::<Lines>() {
-					for line in &*lines.data.lock() {
-						for point in &line.points {
-							bounds.grown_point(Vec3::from(point.point));
-						}
-					}
-				}
-				bounds
+				node.get_aspect::<Lines>()
+					.ok()
+					.map(|v| *v.bounds.lock())
+					.unwrap_or_default()
 			});
 
 		let lines = LINES_REGISTRY.add(Lines {
-			space: node.get_aspect::<Spatial>()?.clone(),
+			spatial: node.get_aspect::<Spatial>()?.clone(),
 			data: Mutex::new(lines),
+			gen_mesh: AtomicBool::new(true),
+			entity: OnceLock::new(),
+			bounds: Mutex::new(Aabb::default()),
 		});
 		node.add_aspect_raw(lines.clone());
 
 		Ok(lines)
 	}
-
-	fn draw(&self, token: &MainThreadToken) {
-		let transform_mat = self.space.global_transform();
-		let data = self.data.lock().clone();
-		for line in &data {
-			let mut points: VecDeque<SkLinePoint> = line
-				.points
-				.iter()
-				.map(|p| SkLinePoint {
-					pt: transform_mat.transform_point3(Vec3::from(p.point)).into(),
-					thickness: p.thickness,
-					color: Color128::new(p.color.c.r, p.color.c.g, p.color.c.b, p.color.a).into(),
-				})
-				.collect();
-			if line.cyclic && !points.is_empty() {
-				let first = line.points.first().unwrap();
-				let last = line.points.last().unwrap();
-
-				let color = Color128 {
-					r: first.color.c.r.lerp(last.color.c.r, 0.5),
-					g: first.color.c.g.lerp(last.color.c.g, 0.5),
-					b: first.color.c.b.lerp(last.color.c.b, 0.5),
-					a: first.color.a.lerp(last.color.a, 0.5),
-				};
-				let connect_point = SkLinePoint {
-					pt: transform_mat
-						.transform_point3(Vec3::from(first.point).lerp(Vec3::from(last.point), 0.5))
-						.into(),
-					thickness: (first.thickness + last.thickness) * 0.5,
-					color: color.into(),
-				};
-				points.push_front(connect_point);
-				points.push_back(connect_point);
-			}
-			stereokit_rust::system::Lines::add_list(token, points.make_contiguous());
-		}
-	}
 }
 impl LinesAspect for Lines {
 	fn set_lines(node: Arc<Node>, _calling_client: Arc<Client>, lines: Vec<Line>) -> Result<()> {
 		let lines_aspect = node.get_aspect::<Lines>()?;
 		*lines_aspect.data.lock() = lines;
+		lines_aspect.gen_mesh.store(true, Ordering::Relaxed);
 		Ok(())
 	}
 }
@@ -92,13 +260,3 @@ impl Drop for Lines {
 		LINES_REGISTRY.remove(self);
 	}
 }
-
-pub fn draw_all(token: &MainThreadToken) {
-	for lines in LINES_REGISTRY.get_valid_contents() {
-		if let Some(node) = lines.space.node() {
-			if node.enabled() {
-				lines.draw(token);
-			}
-		}
-	}
-}

src/nodes/drawable/mod.rs

@@ -1,6 +1,6 @@
 pub mod lines;
 pub mod model;
-pub mod shaders;
+// pub mod shaders;
 pub mod text;
 
 use self::{lines::Lines, model::Model, text::Text};
@@ -8,46 +8,13 @@ use super::{
 	Aspect, AspectIdentifier, Node,
 	spatial::{Spatial, Transform},
 };
-use crate::{DEFAULT_SKYLIGHT, nodes::spatial::SPATIAL_ASPECT_ALIAS_INFO};
-use crate::{
-	DEFAULT_SKYTEX,
-	core::{client::Client, error::Result, resource::get_resource_file},
-};
+use crate::core::{client::Client, error::Result, resource::get_resource_file};
+use crate::nodes::spatial::SPATIAL_ASPECT_ALIAS_INFO;
 use color_eyre::eyre::eyre;
 use model::ModelPart;
 use parking_lot::Mutex;
 use stardust_xr::values::ResourceID;
 use std::{ffi::OsStr, path::PathBuf, sync::Arc};
-use stereokit_rust::{sk::MainThreadToken, system::Renderer, tex::SHCubemap};
-
-// #[instrument(level = "debug", skip(sk))]
-pub fn draw(token: &MainThreadToken) {
-	lines::draw_all(token);
-	model::draw_all(token);
-	text::draw_all(token);
-	match QUEUED_SKYTEX.lock().take() {
-		Some(Some(skytex)) => {
-			if let Ok(skytex) = SHCubemap::from_cubemap(skytex, true, 100) {
-				Renderer::skytex(skytex.tex);
-			}
-		}
-		Some(None) => {
-			Renderer::skytex(DEFAULT_SKYTEX.get().unwrap());
-		}
-		None => {}
-	}
-	match QUEUED_SKYLIGHT.lock().take() {
-		Some(Some(skylight)) => {
-			if let Ok(skylight) = SHCubemap::from_cubemap(skylight, true, 100) {
-				Renderer::skylight(skylight.sh);
-			}
-		}
-		Some(None) => {
-			Renderer::skylight(*DEFAULT_SKYLIGHT.get().unwrap());
-		}
-		None => {}
-	}
-}
 
 static QUEUED_SKYLIGHT: Mutex<Option<Option<PathBuf>>> = Mutex::new(None);
 static QUEUED_SKYTEX: Mutex<Option<Option<PathBuf>>> = Mutex::new(None);

src/nodes/drawable/model.rs

@@ -1,314 +1,498 @@
 use super::{MODEL_PART_ASPECT_ALIAS_INFO, MaterialParameter, ModelAspect, ModelPartAspect};
-use crate::bail;
+use crate::core::bevy_channel::{BevyChannel, BevyChannelReader};
 use crate::core::client::Client;
+use crate::core::color::ColorConvert as _;
+use crate::core::entity_handle::EntityHandle;
 use crate::core::error::Result;
 use crate::core::registry::Registry;
 use crate::core::resource::get_resource_file;
 use crate::nodes::Node;
 use crate::nodes::alias::{Alias, AliasList};
-use crate::nodes::spatial::Spatial;
+use crate::nodes::spatial::{Spatial, SpatialNode};
+use crate::{BevyMaterial, bail};
+use bevy::prelude::*;
+use bevy::render::primitives::Aabb;
 use color_eyre::eyre::eyre;
-use glam::{Mat4, Vec2, Vec3};
 use parking_lot::Mutex;
-use rustc_hash::FxHashMap;
+use rustc_hash::{FxHashMap, FxHasher};
 use stardust_xr::values::ResourceID;
 use std::ffi::OsStr;
 use std::hash::{Hash, Hasher};
-use std::sync::{Arc, LazyLock, OnceLock, Weak};
-use stereokit_rust::material::Transparency;
-use stereokit_rust::maths::Bounds;
-use stereokit_rust::sk::MainThreadToken;
-use stereokit_rust::{material::Material, model::Model as SKModel, tex::Tex, util::Color128};
+use std::path::PathBuf;
+use std::sync::{Arc, OnceLock, Weak};
 
-pub struct MaterialWrapper(pub Material);
-impl Drop for MaterialWrapper {
-	fn drop(&mut self) {
-		MATERIAL_REGISTRY.remove(self);
+static LOAD_MODEL: BevyChannel<(Arc<Model>, PathBuf)> = BevyChannel::new();
+
+pub struct ModelNodePlugin;
+impl Plugin for ModelNodePlugin {
+	fn build(&self, app: &mut App) {
+		LOAD_MODEL.init(app);
+		app.init_resource::<MaterialRegistry>();
+		app.add_systems(Update, load_models);
+		app.add_systems(
+			PostUpdate,
+			(gen_model_parts, apply_materials/* , update_visibillity */).chain(),
+		);
 	}
 }
 
-impl Hash for MaterialWrapper {
-	fn hash<H: Hasher>(&self, state: &mut H) {
-		self.0.get_shader().0.as_ptr().hash(state);
-		for param in self.0.get_all_param_info() {
-			param.name.hash(state);
-			param.to_string().hash(state);
-		}
-		self.0.get_chain().map(MaterialWrapper).hash(state)
-	}
-}
-impl PartialEq for MaterialWrapper {
-	fn eq(&self, other: &Self) -> bool {
-		if self.0.get_shader().0.as_ptr() != other.0.get_shader().0.as_ptr() {
-			return false;
-		}
-		if self.0.get_all_param_info().count() != other.0.get_all_param_info().count() {
-			return false;
-		}
-		for self_param in self.0.get_all_param_info() {
-			let Some(other_param) = other
-				.0
-				.get_all_param_info()
-				.get_data(self_param.get_name(), self_param.get_type())
-			else {
-				return false;
-			};
-			if self_param.to_string() != other_param.to_string() {
-				return false;
+#[derive(Component)]
+struct ModelNode(Weak<Model>);
+
+fn update_visibillity(mut cmds: Commands) {
+	for model in MODEL_REGISTRY.get_valid_contents().into_iter() {
+		let Some(entity) = model.bevy_scene_entity.get() else {
+			continue;
+		};
+		match model.spatial.node().map(|n| n.enabled()).unwrap_or(false) {
+			true => {
+				cmds.entity(entity.0)
+					.insert_recursive::<Children>(Visibility::Visible);
+			}
+			false => {
+				cmds.entity(entity.0)
+					.insert_recursive::<Children>(Visibility::Hidden);
 			}
 		}
-		self.0.get_chain().map(MaterialWrapper) == other.0.get_chain().map(MaterialWrapper)
 	}
 }
-impl Eq for MaterialWrapper {}
-unsafe impl Send for MaterialWrapper {}
-unsafe impl Sync for MaterialWrapper {}
 
-#[derive(Default)]
-struct MaterialRegistry(Mutex<FxHashMap<u64, Weak<MaterialWrapper>>>);
-impl MaterialRegistry {
-	fn add_or_get(&self, material: Arc<MaterialWrapper>) -> Arc<MaterialWrapper> {
-		let hash = {
-			use std::hash::{Hash, Hasher};
-			let mut hasher = std::collections::hash_map::DefaultHasher::new();
-			material.hash(&mut hasher);
-			hasher.finish()
+fn load_models(
+	asset_server: Res<AssetServer>,
+	mut cmds: Commands,
+	mut mpsc_receiver: ResMut<BevyChannelReader<(Arc<Model>, PathBuf)>>,
+) {
+	while let Some((model, path)) = mpsc_receiver.read() {
+		// idk of the asset label is the correct approach here
+		let handle = asset_server.load(GltfAssetLabel::Scene(0).from_asset(path));
+		let entity = cmds
+			.spawn((
+				Name::new("ModelNode"),
+				SceneRoot(handle),
+				ModelNode(Arc::downgrade(&model)),
+				SpatialNode(Arc::downgrade(&model.spatial)),
+			))
+			.id();
+		model.bevy_scene_entity.set(entity.into()).unwrap();
+	}
+}
+
+fn apply_materials(
+	mut query: Query<&mut MeshMaterial3d<BevyMaterial>>,
+	mut material_registry: ResMut<MaterialRegistry>,
+	asset_server: Res<AssetServer>,
+	mut materials: ResMut<Assets<BevyMaterial>>,
+) -> bevy::prelude::Result {
+	for model_part in MODEL_REGISTRY
+		.get_valid_contents()
+		.iter()
+		.filter_map(|p| p.parts.get())
+		.flatten()
+	{
+		let Ok(mut mesh_mat) = query.get_mut(**model_part.mesh_entity.get().unwrap()) else {
+			continue;
 		};
-
-		let mut lock = self.0.lock();
-		if let Some(mat) = lock.get(&hash) {
-			if let Some(mat) = mat.upgrade() {
-				return mat;
-			}
+		if let Some(material) = model_part.pending_material_replacement.lock().take() {
+			let pbr_mat = material.to_pbr_mat(&asset_server);
+			let handle = material_registry.get_handle(pbr_mat, &mut materials);
+			mesh_mat.0 = handle;
+		}
+		for (param_name, param) in model_part.pending_material_parameters.lock().drain() {
+			let mut new_mat = materials.get(&mesh_mat.0).unwrap().clone();
+			param.apply_to_material(
+				&model_part.space.node().unwrap().get_client().unwrap(),
+				&mut new_mat,
+				&param_name,
+				&asset_server,
+			);
+			let handle = material_registry.get_handle(new_mat, &mut materials);
+			mesh_mat.0 = handle;
 		}
-
-		lock.insert(hash, Arc::downgrade(&material));
-		material
 	}
-	fn remove(&self, material: &MaterialWrapper) {
-		let hash = {
-			use std::hash::{Hash, Hasher};
-			let mut hasher = std::collections::hash_map::DefaultHasher::new();
-			material.hash(&mut hasher);
-			hasher.finish()
+
+	Ok(())
+}
+
+fn gen_model_parts(
+	scenes: Res<Assets<Scene>>,
+	query: Query<(&SceneRoot, &ModelNode, &Children)>,
+	children_query: Query<&Children>,
+	part_query: Query<(&Name, Option<&Children>, &Transform), Without<Mesh3d>>,
+	part_mesh_query: Query<(&Transform, &Aabb), With<Mesh3d>>,
+	has_mesh: Query<Has<Mesh3d>>,
+	mut cmds: Commands,
+) {
+	for (scene_root, model_node, model_children) in query.iter() {
+		let Some(model) = model_node.0.upgrade() else {
+			continue;
 		};
-		let mut lock = self.0.lock();
-		lock.remove(&hash);
+		if model.parts.get().is_some() {
+			continue;
+		}
+		if scenes.get(scene_root.0.id()).is_none() {
+			continue;
+		}
+		let mut parts = Vec::new();
+		for entity in model_children
+			.iter()
+			.filter_map(|e| children_query.get(e).ok())
+			.flat_map(|c| c.iter())
+		{
+			gen_path(
+				entity,
+				&part_query,
+				None,
+				&mut |entity, name, transform, parent, children| {
+					let path = parent
+						.as_ref()
+						.map(|p| format!("{}/{}", &p.path, name.as_str()))
+						.unwrap_or_else(|| name.to_string());
+					let parent_spatial = parent
+						.as_ref()
+						.map(|p| p.space.clone())
+						.unwrap_or_else(|| model.spatial.clone());
+					let client = model.spatial.node()?.get_client()?;
+					let (spatial, model_part) =
+						match model.pre_bound_parts.lock().iter().find(|v| v.path == path) {
+							None => {
+								let node =
+									client.scenegraph.add_node(Node::generate(&client, false));
+								let spatial = Spatial::add_to(
+									&node,
+									Some(parent_spatial),
+									transform.compute_matrix(),
+									false,
+								);
+								let model_part = node.add_aspect(ModelPart {
+									entity: OnceLock::new(),
+									mesh_entity: OnceLock::new(),
+									path,
+									space: spatial.clone(),
+									_model: Arc::downgrade(&model),
+									pending_material_parameters: Mutex::default(),
+									pending_material_replacement: Mutex::default(),
+									aliases: AliasList::default(),
+									bounds: OnceLock::new(),
+								});
+								(spatial, model_part)
+							}
+							Some(part) => {
+								part.space
+									.set_spatial_parent(Some(&parent_spatial))
+									.unwrap();
+								(part.space.clone(), part.clone())
+							}
+						};
+					let aabb = Aabb::enclosing(
+						children
+							.iter()
+							.flat_map(|v| v.iter())
+							.filter_map(|e| part_mesh_query.get(e).ok())
+							.flat_map(|(transform, aabb)| {
+								[
+									transform.transform_point(aabb.min().into()),
+									transform.transform_point(aabb.max().into()),
+								]
+							}),
+					)
+					.unwrap_or_default();
+					_ = spatial.bounding_box_calc.set(move |n| {
+						n.get_aspect::<ModelPart>()
+							.ok()
+							.and_then(|v| v.bounds.get().copied())
+							.unwrap_or_default()
+					});
+					cmds.entity(entity)
+						.insert(SpatialNode(Arc::downgrade(&spatial)));
+					let mesh_entity = children_query
+						.get(entity)
+						.iter()
+						.flat_map(|v| v.iter())
+						.find(|e| has_mesh.get(*e).unwrap_or(false))?;
+					_ = model_part.bounds.set(aabb);
+					_ = model_part.entity.set(entity.into());
+					_ = model_part.mesh_entity.set(mesh_entity.into());
+					parts.push(model_part.clone());
+					Some(model_part)
+				},
+			);
+		}
+		_ = model.parts.set(parts);
 	}
 }
 
-static MATERIAL_REGISTRY: LazyLock<MaterialRegistry> = LazyLock::new(MaterialRegistry::default);
+fn gen_path(
+	current_entity: Entity,
+	part_query: &Query<(&Name, Option<&Children>, &Transform), Without<Mesh3d>>,
+	parent: Option<Arc<ModelPart>>,
+	func: &mut dyn FnMut(
+		Entity,
+		&Name,
+		&Transform,
+		Option<Arc<ModelPart>>,
+		Option<&Children>,
+	) -> Option<Arc<ModelPart>>,
+) {
+	let Ok((name, children, transform)) = part_query.get(current_entity) else {
+		return;
+	};
+	let Some(parent) = func(current_entity, name, transform, parent, children) else {
+		return;
+	};
+	for e in children.iter().flat_map(|c| c.iter()) {
+		gen_path(e, part_query, Some(parent.clone()), func);
+	}
+}
+
+#[derive(PartialEq, Deref, DerefMut, Clone, Copy, Eq, PartialOrd, Ord, Hash)]
+struct HashedPbrMaterial(u64);
+impl HashedPbrMaterial {
+	fn new(material: &BevyMaterial) -> Self {
+		let mut hasher = FxHasher::default();
+		Self::hash_pbr_mat(material, &mut hasher);
+		Self(hasher.finish())
+	}
+	fn hash_pbr_mat<H: Hasher>(mat: &BevyMaterial, state: &mut H) {
+		hash_color(mat.base_color, state);
+		hash_color(mat.emissive.into(), state);
+		state.write_u32(mat.metallic.to_bits());
+		state.write_u32(mat.perceptual_roughness.to_bits());
+		match mat.alpha_mode {
+			AlphaMode::Opaque => state.write_u8(0),
+			AlphaMode::Mask(v) => {
+				state.write_u8(1);
+				state.write_u32(v.to_bits());
+			}
+			AlphaMode::Blend => state.write_u8(2),
+			AlphaMode::Premultiplied => state.write_u8(3),
+			AlphaMode::AlphaToCoverage => state.write_u8(4),
+			AlphaMode::Add => state.write_u8(5),
+			AlphaMode::Multiply => state.write_u8(6),
+		}
+		state.write_u8(mat.double_sided as u8);
+		mat.base_color_texture.hash(state);
+		mat.emissive_texture.hash(state);
+		mat.metallic_roughness_texture.hash(state);
+		mat.occlusion_texture.hash(state);
+		// should always be the same, TODO: make the spherical harmonics buffer a per mesh instance thing
+		// mat.spherical_harmonics.hash(state);
+	}
+}
+fn hash_color<H: Hasher>(color: Color, state: &mut H) {
+	match color {
+		Color::Srgba(srgba) => {
+			state.write_u8(0);
+			state.write(&srgba.to_u8_array());
+		}
+		Color::LinearRgba(linear_rgba) => {
+			state.write_u8(1);
+			state.write(&linear_rgba.to_u8_array());
+		}
+		Color::Hsla(hsla) => {
+			state.write_u8(2);
+			hsla.to_f32_array()
+				.iter()
+				.for_each(|v| state.write_u32(v.to_bits()));
+		}
+		Color::Hsva(hsva) => {
+			state.write_u8(3);
+			hsva.to_f32_array()
+				.iter()
+				.for_each(|v| state.write_u32(v.to_bits()));
+		}
+		Color::Hwba(hwba) => {
+			state.write_u8(4);
+			hwba.to_f32_array()
+				.iter()
+				.for_each(|v| state.write_u32(v.to_bits()));
+		}
+		Color::Laba(laba) => {
+			state.write_u8(5);
+			laba.to_f32_array()
+				.iter()
+				.for_each(|v| state.write_u32(v.to_bits()));
+		}
+		Color::Lcha(lcha) => {
+			state.write_u8(6);
+			lcha.to_f32_array()
+				.iter()
+				.for_each(|v| state.write_u32(v.to_bits()));
+		}
+		Color::Oklaba(oklaba) => {
+			state.write_u8(7);
+			oklaba
+				.to_f32_array()
+				.iter()
+				.for_each(|v| state.write_u32(v.to_bits()));
+		}
+		Color::Oklcha(oklcha) => {
+			state.write_u8(8);
+			oklcha
+				.to_f32_array()
+				.iter()
+				.for_each(|v| state.write_u32(v.to_bits()));
+		}
+		Color::Xyza(xyza) => {
+			state.write_u8(9);
+			xyza.to_f32_array()
+				.iter()
+				.for_each(|v| state.write_u32(v.to_bits()));
+		}
+	}
+}
 static MODEL_REGISTRY: Registry<Model> = Registry::new();
-static HOLDOUT_MATERIAL: OnceLock<Arc<MaterialWrapper>> = OnceLock::new();
 
 impl MaterialParameter {
-	fn apply_to_material(&self, client: &Client, material: &Material, parameter_name: &str) {
-		let mut params = material.get_all_param_info();
+	fn apply_to_material(
+		&self,
+		client: &Client,
+		mat: &mut BevyMaterial,
+		parameter_name: &str,
+		asset_server: &AssetServer,
+	) {
 		match self {
-			MaterialParameter::Bool(val) => {
-				params.set_bool(parameter_name, *val);
+			MaterialParameter::Bool(val) => match parameter_name {
+				"double_sided" => mat.double_sided = *val,
+				v => {
+					error!("unknown param_name ({v}) for color")
+				}
+			},
+			MaterialParameter::Int(_val) => {
+				// nothing uses an int
 			}
-			MaterialParameter::Int(val) => {
-				params.set_int(parameter_name, &[*val]);
-			}
-			MaterialParameter::UInt(val) => {
-				params.set_uint(parameter_name, &[*val]);
+			MaterialParameter::UInt(_val) => {
+				// nothing uses an uint
 			}
 			MaterialParameter::Float(val) => {
-				params.set_float(parameter_name, *val);
+				match parameter_name {
+					"metallic" => mat.metallic = *val,
+					"roughness" => mat.perceptual_roughness = *val,
+					// we probably don't want to expose tex_scale
+					// "tex_scale" => mat.tex_scale = *val,
+					v => {
+						error!("unknown param_name ({v}) for float")
+					}
+				}
 			}
-			MaterialParameter::Vec2(val) => {
-				params.set_vec2(parameter_name, Vec2::from(*val));
+			MaterialParameter::Vec2(_val) => {
+				// nothing uses a Vec2
 			}
-			MaterialParameter::Vec3(val) => {
-				params.set_vec3(parameter_name, Vec3::from(*val));
-			}
-			MaterialParameter::Color(val) => {
-				params.set_color(
-					parameter_name,
-					Color128::new(val.c.r, val.c.g, val.c.b, val.a),
-				);
+			MaterialParameter::Vec3(_val) => {
+				// nothing uses a Vec3
 			}
+			MaterialParameter::Color(color) => match parameter_name {
+				"color" => mat.base_color = color.to_bevy(),
+				"emission_factor" => mat.emissive = color.to_bevy().to_linear(),
+				v => {
+					error!("unknown param_name ({v}) for color")
+				}
+			},
 			MaterialParameter::Texture(resource) => {
 				let Some(texture_path) =
 					get_resource_file(resource, client, &[OsStr::new("png"), OsStr::new("jpg")])
 				else {
 					return;
 				};
-				if let Ok(tex) = Tex::from_file(texture_path, true, None) {
-					params.set_texture(parameter_name, &tex);
+				let handle = asset_server.load(texture_path);
+				match parameter_name {
+					"diffuse" => mat.base_color_texture = Some(handle),
+					"emission" => mat.emissive_texture = Some(handle),
+					"metal" => mat.metallic_roughness_texture = Some(handle),
+					"occlusion" => mat.occlusion_texture = Some(handle),
+					v => {
+						error!("unknown param_name ({v}) for texture");
+					}
 				}
+				// mat.alpha_mode = AlphaMode::Blend;
 			}
 		}
 	}
 }
 
+pub struct Material {
+	pub color: Color,
+	pub emission_factor: Color,
+	pub metallic: f32,
+	pub roughness: f32,
+	pub alpha_mode: AlphaMode,
+	pub double_sided: bool,
+
+	pub diffuse_texture: Option<PathBuf>,
+	pub emission_texture: Option<PathBuf>,
+	pub metal_texture: Option<PathBuf>,
+	pub occlusion_texture: Option<PathBuf>,
+}
+
+impl Material {
+	fn to_pbr_mat(&self, asset_server: &AssetServer) -> BevyMaterial {
+		BevyMaterial {
+			base_color: self.color,
+			emissive: self.emission_factor.to_linear(),
+			metallic: self.metallic,
+			perceptual_roughness: self.roughness,
+			alpha_mode: self.alpha_mode,
+			double_sided: self.double_sided,
+			base_color_texture: self
+				.diffuse_texture
+				.as_ref()
+				.map(|p| asset_server.load(p.as_path())),
+			emissive_texture: self
+				.emission_texture
+				.as_ref()
+				.map(|p| asset_server.load(p.as_path())),
+			metallic_roughness_texture: self
+				.metal_texture
+				.as_ref()
+				.map(|p| asset_server.load(p.as_path())),
+			occlusion_texture: self
+				.occlusion_texture
+				.as_ref()
+				.map(|p| asset_server.load(p.as_path())),
+			..Default::default()
+			// spherical_harmonics: bevy_sk::skytext::SPHERICAL_HARMONICS_HANDLE,
+		}
+	}
+}
+
 pub struct ModelPart {
-	id: i32,
+	entity: OnceLock<EntityHandle>,
+	mesh_entity: OnceLock<EntityHandle>,
 	path: String,
 	space: Arc<Spatial>,
-	model: Weak<Model>,
-	material: Mutex<Option<Arc<MaterialWrapper>>>,
+	_model: Weak<Model>,
 	pending_material_parameters: Mutex<FxHashMap<String, MaterialParameter>>,
-	pending_material_replacement: Mutex<Option<Arc<MaterialWrapper>>>,
+	pending_material_replacement: Mutex<Option<Material>>,
 	aliases: AliasList,
+	bounds: OnceLock<Aabb>,
 }
 impl ModelPart {
-	fn create_for_model(model: &Arc<Model>, sk_model: &SKModel) {
-		HOLDOUT_MATERIAL.get_or_init(|| {
-			let mut mat = Material::copy(&Material::unlit());
-			mat.transparency(Transparency::None);
-			mat.color_tint(Color128::BLACK_TRANSPARENT);
-			Arc::new(MaterialWrapper(mat))
-		});
-
-		let nodes = sk_model.get_nodes();
-		for part in nodes.all() {
-			ModelPart::create(model, &part);
-		}
-	}
-
-	fn create(model: &Arc<Model>, part: &stereokit_rust::model::ModelNode) -> Option<Arc<Self>> {
-		let mut parts = model.parts.lock();
-		let parent_part = part
-			.get_parent()
-			.and_then(|part| parts.iter().find(|p| p.id == *part.get_id()));
-
-		let stardust_model_part = model.space.node()?;
-		let client = stardust_model_part.get_client()?;
-		let mut part_path = parent_part
-			.map(|n| n.path.clone() + "/")
-			.unwrap_or_default();
-		part_path += part.get_name().unwrap();
-
-		let node = client.scenegraph.add_node(Node::generate(&client, false));
-		let spatial_parent = parent_part
-			.map(|n| n.space.clone())
-			.unwrap_or_else(|| model.space.clone());
-
-		let local_transform = unsafe { part.get_local_transform().m };
-		let space = Spatial::add_to(
-			&node,
-			Some(spatial_parent),
-			Mat4::from_cols_array(&local_transform),
-			false,
-		);
-
-		let _ = space.bounding_box_calc.set(|node| {
-			let Ok(model_part) = node.get_aspect::<ModelPart>() else {
-				return Bounds::default();
-			};
-			let Some(model) = model_part.model.upgrade() else {
-				return Bounds::default();
-			};
-			let Some(sk_model) = model.sk_model.get() else {
-				return Bounds::default();
-			};
-			let model_nodes = sk_model.get_nodes();
-			let Some(model_node) = model_nodes.get_index(model_part.id) else {
-				return Bounds::default();
-			};
-			let Some(sk_mesh) = model_node.get_mesh() else {
-				return Bounds::default();
-			};
-			sk_mesh.get_bounds()
-		});
-
-		let model_part = Arc::new(ModelPart {
-			id: *part.get_id(),
-			path: part_path,
-			space,
-			model: Arc::downgrade(model),
-			pending_material_parameters: Mutex::new(FxHashMap::default()),
-			pending_material_replacement: Mutex::new(None),
-			aliases: AliasList::default(),
-			material: Mutex::new(part.get_material().map(MaterialWrapper).map(Arc::new)),
-		});
-		node.add_aspect_raw(model_part.clone());
-		parts.push(model_part.clone());
-		Some(model_part)
-	}
-
-	pub fn replace_material(&self, replacement: Arc<MaterialWrapper>) {
-		let shared_material = MATERIAL_REGISTRY.add_or_get(replacement);
+	pub fn replace_material(&self, replacement: Material) {
 		self.pending_material_replacement
 			.lock()
-			.replace(shared_material);
+			.replace(replacement);
 	}
-	/// only to be run on the main thread
-	pub fn replace_material_now(&self, replacement: &Material) {
-		let Some(model) = self.model.upgrade() else {
-			return;
-		};
-		let Some(sk_model) = model.sk_model.get() else {
-			return;
-		};
-		let nodes = sk_model.get_nodes();
-		let Some(mut part) = nodes.get_index(self.id) else {
-			return;
-		};
-		let shared_material =
-			MATERIAL_REGISTRY.add_or_get(Arc::new(MaterialWrapper(replacement.copy())));
-
-		let mut lock = self.material.lock();
-		part.material(&shared_material.0);
-		lock.replace(shared_material);
-	}
-
-	fn update(&self) {
-		let Some(model) = self.model.upgrade() else {
-			return;
-		};
-		let Some(sk_model) = model.sk_model.get() else {
-			return;
-		};
-		let Some(node) = model.space.node() else {
-			return;
-		};
-		let nodes = sk_model.get_nodes();
-		let Some(mut part) = nodes.get_index(self.id) else {
-			return;
-		};
-		part.model_transform(Spatial::space_to_space_matrix(
-			Some(&self.space),
-			Some(&model.space),
-		));
-
-		let Some(client) = node.get_client() else {
-			return;
-		};
-
-		if let Some(material_replacement) = self.pending_material_replacement.lock().take() {
-			let mut lock = self.material.lock();
-			part.material(&material_replacement.0);
-			lock.replace(material_replacement);
-		}
-
-		'mat_params: {
-			let mut material_parameters = self.pending_material_parameters.lock();
-			if !material_parameters.is_empty() {
-				let Some(material) = part.get_material() else {
-					break 'mat_params;
-				};
-				let new_material = material.copy();
-				for (parameter_name, parameter_value) in material_parameters.drain() {
-					parameter_value.apply_to_material(&client, &new_material, &parameter_name);
-				}
-
-				let shared_material =
-					MATERIAL_REGISTRY.add_or_get(Arc::new(MaterialWrapper(new_material)));
-				let mut lock = self.material.lock();
-				part.material(&shared_material.0);
-				lock.replace(shared_material);
-			}
-		}
+	pub fn set_material_parameter(&self, parameter_name: String, value: MaterialParameter) {
+		self.pending_material_parameters
+			.lock()
+			.insert(parameter_name, value);
 	}
 }
 impl ModelPartAspect for ModelPart {
 	#[doc = "Set this model part's material to one that cuts a hole in the world. Often used for overlays/passthrough where you want to show the background through an object."]
 	fn apply_holdout_material(node: Arc<Node>, _calling_client: Arc<Client>) -> Result<()> {
 		let model_part = node.get_aspect::<ModelPart>()?;
-		model_part.replace_material(HOLDOUT_MATERIAL.get().unwrap().clone());
+		model_part.replace_material(Material {
+			color: Color::BLACK.with_alpha(0.0),
+			emission_factor: Color::BLACK,
+			metallic: 0.0,
+			roughness: 1.0,
+			alpha_mode: AlphaMode::Opaque,
+			double_sided: false,
+			diffuse_texture: None,
+			emission_texture: None,
+			metal_texture: None,
+			occlusion_texture: None,
+		});
 		Ok(())
 	}
 
@@ -320,20 +504,41 @@ impl ModelPartAspect for ModelPart {
 		value: MaterialParameter,
 	) -> Result<()> {
 		let model_part = node.get_aspect::<ModelPart>()?;
-		model_part
-			.pending_material_parameters
-			.lock()
-			.insert(parameter_name, value);
-
+		model_part.set_material_parameter(parameter_name, value);
 		Ok(())
 	}
 }
+#[derive(Default, Resource)]
+pub struct MaterialRegistry(FxHashMap<HashedPbrMaterial, Handle<BevyMaterial>>);
+impl MaterialRegistry {
+	/// returns strong handle for PbrMaterial elminitating duplications
+	pub fn get_handle(
+		&mut self,
+		material: BevyMaterial,
+		materials: &mut ResMut<Assets<BevyMaterial>>,
+	) -> Handle<BevyMaterial> {
+		let hash = HashedPbrMaterial::new(&material);
+		match self
+			.0
+			.get(&hash)
+			.and_then(|v| materials.get_strong_handle(v.id()))
+		{
+			Some(v) => v,
+			None => {
+				let handle = materials.add(material);
+				self.0.insert(hash, handle.clone_weak());
+				handle
+			}
+		}
+	}
+}
 
 pub struct Model {
-	space: Arc<Spatial>,
+	spatial: Arc<Spatial>,
 	_resource_id: ResourceID,
-	sk_model: OnceLock<SKModel>,
-	parts: Mutex<Vec<Arc<ModelPart>>>,
+	bevy_scene_entity: OnceLock<EntityHandle>,
+	parts: OnceLock<Vec<Arc<ModelPart>>>,
+	pre_bound_parts: Mutex<Vec<Arc<ModelPart>>>,
 }
 impl Model {
 	pub fn add_to(node: &Arc<Node>, resource_id: ResourceID) -> Result<Arc<Model>> {
@@ -345,44 +550,67 @@ impl Model {
 		.ok_or_else(|| eyre!("Resource not found"))?;
 
 		let model = Arc::new(Model {
-			space: node.get_aspect::<Spatial>().unwrap().clone(),
+			spatial: node.get_aspect::<Spatial>().unwrap().clone(),
 			_resource_id: resource_id,
-			sk_model: OnceLock::new(),
-			parts: Mutex::new(Vec::default()),
+			bevy_scene_entity: OnceLock::new(),
+			pre_bound_parts: Mutex::default(),
+			parts: OnceLock::new(),
 		});
+		LOAD_MODEL
+			.send((model.clone(), pending_model_path))
+			.unwrap();
 		MODEL_REGISTRY.add_raw(&model);
 
-		// technically doing this in anything but the main thread isn't a good idea but dangit we need those model nodes ASAP
-		let sk_model = SKModel::copy(SKModel::from_file(
-			pending_model_path.to_str().unwrap(),
-			None,
-		)?);
-		ModelPart::create_for_model(&model, &sk_model);
-		let _ = model.sk_model.set(sk_model);
 		node.add_aspect_raw(model.clone());
 		Ok(model)
 	}
-
-	fn draw(&self, token: &MainThreadToken) {
-		let Some(sk_model) = self.sk_model.get() else {
-			return;
-		};
-		let parts = self.parts.lock();
-		for model_node in &*parts {
-			model_node.update();
-		}
-		drop(parts);
-
-		if let Some(node) = self.space.node() {
-			if node.enabled() {
-				sk_model.draw(token, self.space.global_transform(), None, None);
+	pub fn get_model_part(self: &Arc<Self>, part_path: String) -> Result<Arc<ModelPart>> {
+		let part = match self
+			.parts
+			.get()
+			.map(|v| v.iter().find(|p| p.path == part_path))
+		{
+			Some(Some(part)) => part.clone(),
+			Some(None) => {
+				let paths = self
+					.parts
+					.get()
+					.unwrap()
+					.iter()
+					.map(|p| &p.path)
+					.collect::<Vec<_>>();
+				bail!(
+					"Couldn't find model part at path {part_path}, all available paths: {paths:?}",
+				);
 			}
-		}
+			None => {
+				// TODO: this could be a denail of service vector
+				let client = self.spatial.node().unwrap().get_client().unwrap();
+				let part_node = client.scenegraph.add_node(Node::generate(&client, false));
+				let spatial = Spatial::add_to(
+					&part_node,
+					Some(self.spatial.clone()),
+					Mat4::IDENTITY,
+					false,
+				);
+				let part = part_node.add_aspect(ModelPart {
+					entity: OnceLock::new(),
+					mesh_entity: OnceLock::new(),
+					path: part_path,
+					space: spatial,
+					_model: Arc::downgrade(self),
+					pending_material_parameters: Mutex::default(),
+					pending_material_replacement: Mutex::default(),
+					aliases: AliasList::default(),
+					bounds: OnceLock::new(),
+				});
+				self.pre_bound_parts.lock().push(part.clone());
+				part
+			}
+		};
+		Ok(part)
 	}
 }
-// TODO: proper hread safety in stereokit_rust (probably just bind stereokit directly)
-unsafe impl Send for Model {}
-unsafe impl Sync for Model {}
 impl ModelAspect for Model {
 	#[doc = "Bind a model part to the node with the ID input."]
 	fn bind_model_part(
@@ -392,11 +620,7 @@ impl ModelAspect for Model {
 		part_path: String,
 	) -> Result<()> {
 		let model = node.get_aspect::<Model>()?;
-		let parts = model.parts.lock();
-		let Some(part) = parts.iter().find(|p| p.path == part_path) else {
-			let paths = parts.iter().map(|p| &p.path).collect::<Vec<_>>();
-			bail!("Couldn't find model part at path {part_path}, all available paths: {paths:?}",);
-		};
+		let part = model.get_model_part(part_path)?;
 		Alias::create_with_id(
 			&part.space.node().unwrap(),
 			&calling_client,
@@ -412,9 +636,3 @@ impl Drop for Model {
 		MODEL_REGISTRY.remove(self);
 	}
 }
-
-pub fn draw_all(token: &MainThreadToken) {
-	for model in MODEL_REGISTRY.get_valid_contents() {
-		model.draw(token);
-	}
-}

src/nodes/drawable/text.rs

@@ -1,48 +1,198 @@
 use crate::{
+	BevyMaterial,
 	core::{
-		client::Client, destroy_queue, error::Result, registry::Registry,
+		bevy_channel::{BevyChannel, BevyChannelReader},
+		client::Client,
+		color::ColorConvert,
+		entity_handle::EntityHandle,
+		error::Result,
+		registry::Registry,
 		resource::get_resource_file,
 	},
-	nodes::{Node, spatial::Spatial},
+	nodes::{
+		Node,
+		drawable::XAlign,
+		spatial::{Spatial, SpatialNode},
+	},
+};
+use bevy::{platform::collections::HashMap, prelude::*, render::mesh::MeshAabb};
+use bevy_mesh_text_3d::{
+	Align, Attrs, MeshTextPlugin, Settings as FontSettings, generate_meshes,
+	text_glyphs::TextGlyphs,
 };
 use color_eyre::eyre::eyre;
-use glam::{Mat4, Vec2, vec3};
+use core::f32;
+use cosmic_text::Metrics;
 use parking_lot::Mutex;
-use std::{
-	ffi::OsStr,
-	path::PathBuf,
-	sync::{Arc, OnceLock},
-};
-use stereokit_rust::{
-	font::Font,
-	sk::MainThreadToken,
-	system::{TextAlign, TextFit, TextStyle as SkTextStyle},
-	util::{Color32, Color128},
-};
+use std::{ffi::OsStr, mem, path::PathBuf, sync::Arc};
 
-use super::{TextAspect, TextStyle};
+static SPAWN_TEXT: BevyChannel<Arc<Text>> = BevyChannel::new();
 
-static TEXT_REGISTRY: Registry<Text> = Registry::new();
+pub struct TextNodePlugin;
 
-fn convert_align(x_align: super::XAlign, y_align: super::YAlign) -> TextAlign {
-	match (x_align, y_align) {
-		(super::XAlign::Left, super::YAlign::Top) => TextAlign::TopLeft,
-		(super::XAlign::Left, super::YAlign::Center) => TextAlign::CenterLeft,
-		(super::XAlign::Left, super::YAlign::Bottom) => TextAlign::BottomLeft,
-		(super::XAlign::Center, super::YAlign::Top) => TextAlign::Center,
-		(super::XAlign::Center, super::YAlign::Center) => TextAlign::Center,
-		(super::XAlign::Center, super::YAlign::Bottom) => TextAlign::BottomCenter,
-		(super::XAlign::Right, super::YAlign::Top) => TextAlign::TopRight,
-		(super::XAlign::Right, super::YAlign::Center) => TextAlign::CenterRight,
-		(super::XAlign::Right, super::YAlign::Bottom) => TextAlign::BottomRight,
+impl Plugin for TextNodePlugin {
+	fn build(&self, app: &mut App) {
+		// Text init stuff
+		// 1.0 for font size in meters
+		app.add_plugins(MeshTextPlugin::new(1.0));
+		app.world_mut()
+			.resource_mut::<FontSettings>()
+			.font_system
+			.db_mut()
+			.load_system_fonts();
+
+		SPAWN_TEXT.init(app);
+		app.init_resource::<MaterialRegistry>();
+		app.add_systems(Update, spawn_text);
 	}
 }
 
-pub struct Text {
-	space: Arc<Spatial>,
-	font_path: Option<PathBuf>,
-	style: OnceLock<SkTextStyle>,
+fn spawn_text(
+	mut mpsc: ResMut<BevyChannelReader<Arc<Text>>>,
+	mut cmds: Commands,
+	mut font_settings: ResMut<FontSettings>,
+	mut material_registry: ResMut<MaterialRegistry>,
+	mut materials: ResMut<Assets<BevyMaterial>>,
+	mut meshes: ResMut<Assets<Mesh>>,
+	mut font_registry: Local<FontDatabaseRegistry>,
+) {
+	while let Some(text) = mpsc.read() {
+		if let Some(entity) = text.entity.lock().take() {
+			cmds.entity(*entity).despawn();
+		}
+		let style = text.data.lock();
+		let old_db = text.font_path.clone().map(|p| {
+			let db = font_registry.get(p);
+			mem::swap(font_settings.font_system.db_mut(), db);
+			db
+		});
+		let attrs = Attrs::new().weight(cosmic_text::Weight::BOLD);
+		let alignment = Some(match style.text_align_x {
+			super::XAlign::Left => Align::Right,
+			super::XAlign::Center => Align::Center,
+			super::XAlign::Right => Align::Left,
+		});
+		let text_string = text.text.lock().clone();
+		let mut text_glyphs = TextGlyphs::new(
+			Metrics {
+				font_size: style.character_height,
+				line_height: style.character_height,
+			},
+			[(text_string.as_str(), attrs.clone())],
+			&attrs,
+			&mut font_settings.font_system,
+			alignment,
+		);
+		let max_width = style.bounds.as_ref().map(|v| v.bounds.x);
+		let max_height = style.bounds.as_ref().map(|v| v.bounds.x);
+		let (width, _height) =
+			text_glyphs.measure(max_width, max_height, &mut font_settings.font_system);
+		let char_meshes = generate_meshes(
+			bevy_mesh_text_3d::InputText::Simple {
+				text: text_string,
+				material: material_registry.get_handle(
+					BevyMaterial {
+						base_color: style.color.to_bevy(),
+						emissive: Color::WHITE.to_linear(),
+						metallic: 0.0,
+						perceptual_roughness: 1.0,
+						// If alpha is supported on text we need to change this
+						alpha_mode: AlphaMode::Opaque,
+						double_sided: false,
+						..default()
+					},
+					&mut materials,
+				),
+				attrs,
+			},
+			&mut font_settings,
+			bevy_mesh_text_3d::Parameters {
+				extrusion_depth: 0.0,
+				font_size: style.character_height,
+				line_height: style.character_height,
+				alignment,
+				max_width,
+				max_height,
+			},
+			&mut meshes,
+		);
+		if let Some(db) = old_db {
+			mem::swap(font_settings.font_system.db_mut(), db);
+		}
+		let Ok(char_meshes) =
+			char_meshes.inspect_err(|err| error!("unable to create text meshes: {err}"))
+		else {
+			continue;
+		};
+		let dist = char_meshes.iter().fold(f32::MAX, |dist, v| {
+			dist.min(
+				v.transform.translation.x
+					- meshes
+						.get(&v.mesh)
+						.unwrap()
+						.compute_aabb()
+						.unwrap_or_default()
+						.half_extents
+						.x,
+			)
+		});
+		// TODO: text align
+		let letters = char_meshes
+			.into_iter()
+			.map(|v| {
+				cmds.spawn((
+					Mesh3d(v.mesh),
+					MeshMaterial3d(v.material),
+					// rotation is sus, might be related to the gltf coordinate system
+					Transform::from_rotation(Quat::from_rotation_y(f32::consts::PI))
+						* Transform::from_xyz(
+							-dist
+								+ match style.bounds.as_ref().map(|v| v.anchor_align_x) {
+									Some(XAlign::Center) => width * -0.5,
+									Some(XAlign::Right) => -width,
+									Some(XAlign::Left) => 0.0,
+									None => 0.0,
+								},
+							0.0,
+							0.0,
+						) * v.transform,
+				))
+				.id()
+			})
+			.collect::<Vec<_>>();
+		let entity = cmds
+			.spawn((
+				Name::new("TextNode"),
+				SpatialNode(Arc::downgrade(&text.spatial)),
+			))
+			.add_children(&letters)
+			.id();
+		text.entity.lock().replace(EntityHandle(entity));
+	}
+}
 
+#[derive(Default)]
+struct FontDatabaseRegistry(HashMap<PathBuf, cosmic_text::fontdb::Database>);
+impl FontDatabaseRegistry {
+	fn get(&mut self, path: PathBuf) -> &mut cosmic_text::fontdb::Database {
+		self.0.entry(path).or_insert_with_key(|path| {
+			let mut db = cosmic_text::fontdb::Database::new();
+			if let Err(err) = db.load_font_file(path) {
+				error!("unable to load font file {} {err}", path.to_string_lossy());
+			};
+			db
+		})
+	}
+}
+
+use super::{TextAspect, TextStyle, model::MaterialRegistry};
+
+static TEXT_REGISTRY: Registry<Text> = Registry::new();
+
+pub struct Text {
+	spatial: Arc<Spatial>,
+	font_path: Option<PathBuf>,
+	entity: Mutex<Option<EntityHandle>>,
 	text: Mutex<String>,
 	data: Mutex<TextStyle>,
 }
@@ -50,88 +200,20 @@ impl Text {
 	pub fn add_to(node: &Arc<Node>, text: String, style: TextStyle) -> Result<Arc<Text>> {
 		let client = node.get_client().ok_or_else(|| eyre!("Client not found"))?;
 		let text = TEXT_REGISTRY.add(Text {
-			space: node.get_aspect::<Spatial>().unwrap().clone(),
+			spatial: node.get_aspect::<Spatial>().unwrap().clone(),
 			font_path: style.font.as_ref().and_then(|res| {
 				get_resource_file(res, &client, &[OsStr::new("ttf"), OsStr::new("otf")])
 			}),
-			style: OnceLock::new(),
 
+			entity: Mutex::new(None),
 			text: Mutex::new(text),
 			data: Mutex::new(style),
 		});
 		node.add_aspect_raw(text.clone());
+		_ = SPAWN_TEXT.send(text.clone());
 
 		Ok(text)
 	}
-
-	fn draw(&self, token: &MainThreadToken) {
-		let style = self.style.get_or_init(|| {
-			let font = self
-				.font_path
-				.as_deref()
-				.and_then(|path| Font::from_file(path).ok())
-				.unwrap_or_default();
-			SkTextStyle::from_font(font, 1.0, Color32::WHITE)
-		});
-
-		let text = self.text.lock();
-		let data = self.data.lock();
-		let transform = self.space.global_transform()
-			* Mat4::from_scale(vec3(
-				data.character_height,
-				data.character_height,
-				data.character_height,
-			));
-		if let Some(bounds) = &data.bounds {
-			stereokit_rust::system::Text::add_in(
-				token,
-				&*text,
-				transform,
-				Vec2::from(bounds.bounds) / data.character_height,
-				match bounds.fit {
-					super::TextFit::Wrap => TextFit::Wrap,
-					super::TextFit::Clip => TextFit::Clip,
-					super::TextFit::Squeeze => TextFit::Squeeze,
-					super::TextFit::Exact => TextFit::Exact,
-					super::TextFit::Overflow => TextFit::Overflow,
-				},
-				Some(*style),
-				Some(Color128::new(
-					data.color.c.r,
-					data.color.c.g,
-					data.color.c.b,
-					data.color.a,
-				)),
-				data.bounds
-					.as_ref()
-					.map(|b| convert_align(b.anchor_align_x, b.anchor_align_y)),
-				Some(convert_align(data.text_align_x, data.text_align_y)),
-				None,
-				None,
-				None,
-			);
-		} else {
-			stereokit_rust::system::Text::add_at(
-				token,
-				&*text,
-				transform,
-				Some(*style),
-				Some(Color128::new(
-					data.color.c.r,
-					data.color.c.g,
-					data.color.c.b,
-					data.color.a,
-				)),
-				data.bounds
-					.as_ref()
-					.map(|b| convert_align(b.anchor_align_x, b.anchor_align_y)),
-				Some(convert_align(data.text_align_x, data.text_align_y)),
-				None,
-				None,
-				None,
-			);
-		}
-	}
 }
 impl TextAspect for Text {
 	fn set_character_height(
@@ -141,30 +223,19 @@ impl TextAspect for Text {
 	) -> Result<()> {
 		let this_text = node.get_aspect::<Text>()?;
 		this_text.data.lock().character_height = height;
+		_ = SPAWN_TEXT.send(this_text);
 		Ok(())
 	}
 
 	fn set_text(node: Arc<Node>, _calling_client: Arc<Client>, text: String) -> Result<()> {
 		let this_text = node.get_aspect::<Text>()?;
 		*this_text.text.lock() = text;
+		_ = SPAWN_TEXT.send(this_text);
 		Ok(())
 	}
 }
 impl Drop for Text {
 	fn drop(&mut self) {
-		if let Some(style) = self.style.take() {
-			destroy_queue::add(style);
-		}
 		TEXT_REGISTRY.remove(self);
 	}
 }
-
-pub fn draw_all(token: &MainThreadToken) {
-	for text in TEXT_REGISTRY.get_valid_contents() {
-		if let Some(node) = text.space.node() {
-			if node.enabled() {
-				text.draw(token);
-			}
-		}
-	}
-}

src/nodes/input/mod.rs

@@ -6,8 +6,11 @@ mod method;
 mod pointer;
 mod tip;
 
+use bevy::tasks::ComputeTaskPool;
+use bevy::tasks::ParallelSlice;
 pub use handler::*;
 pub use method::*;
+use tracing::debug_span;
 
 use super::Aspect;
 use super::AspectIdentifier;
@@ -119,51 +122,60 @@ pub fn process_input() {
 			};
 			node.enabled()
 		});
-	for handler in INPUT_HANDLER_REGISTRY.get_valid_contents() {
-		for method_alias in handler.method_aliases.get_aliases() {
-			method_alias.set_enabled(false);
-		}
+	INPUT_HANDLER_REGISTRY
+		.get_valid_contents()
+		.into_iter()
+		.par_splat_map(ComputeTaskPool::get(), None, |_, handlers| {
+			for handler in handlers {
+				let _span = debug_span!("handle input handler").entered();
+				for method_alias in handler.method_aliases.get_aliases() {
+					method_alias.set_enabled(false);
+				}
 
-		let Some(handler_node) = handler.spatial.node() else {
-			continue;
-		};
-		if !handler_node.enabled() {
-			continue;
-		}
-		if let Some(handler_field_node) = handler.field.spatial.node() {
-			if !handler_field_node.enabled() {
-				continue;
+				let Some(handler_node) = handler.spatial.node() else {
+					continue;
+				};
+				if !handler_node.enabled() {
+					continue;
+				}
+				if let Some(handler_field_node) = handler.field.spatial.node() {
+					if !handler_field_node.enabled() {
+						continue;
+					}
+				};
+
+				let ser_span = debug_span!("serializing input").entered();
+				let (methods, datas) = methods
+					.clone()
+					// filter out methods without the handler in their handler order
+					.filter(|a| {
+						a.handler_order
+							.lock()
+							.iter()
+							.any(|h| h.ptr_eq(&Arc::downgrade(&handler)))
+					})
+					// filter out methods without the proper alias
+					.filter_map(|m| {
+						Some((
+							handler
+								.method_aliases
+								.get_from_original_node(m.spatial.node.clone())?,
+							m,
+						))
+					})
+					// make sure the input method alias is enabled
+					.inspect(|(a, _)| {
+						a.set_enabled(true);
+					})
+					// serialize the data
+					.map(|(a, m)| (a.clone(), m.serialize(a.get_id(), &handler)))
+					.unzip::<_, _, Vec<_>, Vec<_>>();
+				drop(ser_span);
+
+				let _span = debug_span!("client input").entered();
+				let _ = input_handler_client::input(&handler_node, &methods, &datas);
 			}
-		};
-
-		let (methods, datas) = methods
-			.clone()
-			// filter out methods without the handler in their handler order
-			.filter(|a| {
-				a.handler_order
-					.lock()
-					.iter()
-					.any(|h| h.ptr_eq(&Arc::downgrade(&handler)))
-			})
-			// filter out methods without the proper alias
-			.filter_map(|m| {
-				Some((
-					handler
-						.method_aliases
-						.get_from_original_node(m.spatial.node.clone())?,
-					m,
-				))
-			})
-			// make sure the input method alias is enabled
-			.inspect(|(a, _)| {
-				a.set_enabled(true);
-			})
-			// serialize the data
-			.map(|(a, m)| (a.clone(), m.serialize(a.get_id(), &handler)))
-			.unzip::<_, _, Vec<_>, Vec<_>>();
-
-		let _ = input_handler_client::input(&handler_node, &methods, &datas);
-	}
+		});
 	for method in methods {
 		method.cull_capture_attempts();
 	}

src/nodes/items/camera.rs

@@ -1,3 +1,4 @@
+#![allow(dead_code)]
 use super::{Item, ItemType, create_item_acceptor_flex, register_item_ui_flex};
 use crate::bail;
 use crate::core::error::Result;
@@ -9,10 +10,7 @@ use crate::{
 	core::{client::Client, registry::Registry, scenegraph::MethodResponseSender},
 	nodes::{
 		Message, Node,
-		drawable::{
-			model::{MaterialWrapper, ModelPart},
-			shaders::UNLIT_SHADER_BYTES,
-		},
+		drawable::model::ModelPart,
 		items::TypeInfo,
 		spatial::{Spatial, Transform},
 	},
@@ -24,19 +22,6 @@ use parking_lot::Mutex;
 
 use stardust_xr::schemas::flex::{deserialize, serialize};
 use std::sync::Arc;
-use std::sync::OnceLock;
-use stereokit_rust::{
-	material::{Material, Transparency},
-	shader::Shader,
-	sk::MainThreadToken,
-	system::Renderer,
-	tex::{Tex, TexFormat, TexType},
-	util::Color128,
-};
-
-pub struct TexWrapper(pub Tex);
-unsafe impl Send for TexWrapper {}
-unsafe impl Sync for TexWrapper {}
 
 stardust_xr_server_codegen::codegen_item_camera_protocol!();
 lazy_static! {
@@ -67,8 +52,6 @@ struct FrameInfo {
 pub struct CameraItem {
 	space: Arc<Spatial>,
 	frame_info: Mutex<FrameInfo>,
-	sk_tex: OnceLock<TexWrapper>,
-	sk_mat: OnceLock<Arc<MaterialWrapper>>,
 	applied_to: Registry<ModelPart>,
 	apply_to: Registry<ModelPart>,
 }
@@ -81,8 +64,6 @@ impl CameraItem {
 				proj_matrix,
 				px_size,
 			}),
-			sk_tex: OnceLock::new(),
-			sk_mat: OnceLock::new(),
 			applied_to: Registry::new(),
 			apply_to: Registry::new(),
 		});
@@ -127,41 +108,6 @@ impl CameraItem {
 	pub fn send_acceptor_item_created(&self, node: &Node, item: &Arc<Node>) {
 		let _ = camera_item_acceptor_client::capture_item(node, item);
 	}
-
-	pub fn update(&self, token: &MainThreadToken) {
-		let frame_info = self.frame_info.lock();
-		let sk_tex = self.sk_tex.get_or_init(|| {
-			TexWrapper(Tex::gen_color(
-				Color128::default(),
-				frame_info.px_size.x as i32,
-				frame_info.px_size.y as i32,
-				TexType::Rendertarget,
-				TexFormat::RGBA32Linear,
-			))
-		});
-		let sk_mat = self.sk_mat.get_or_init(|| {
-			let shader = Shader::from_memory(UNLIT_SHADER_BYTES).unwrap();
-			let mut mat = Material::new(&shader, None);
-			mat.get_all_param_info().set_texture("diffuse", &sk_tex.0);
-			mat.transparency(Transparency::Blend);
-			Arc::new(MaterialWrapper(mat))
-		});
-		for model_part in self.apply_to.take_valid_contents() {
-			model_part.replace_material(sk_mat.clone())
-		}
-
-		if !self.applied_to.is_empty() {
-			Renderer::render_to(
-				token,
-				&sk_tex.0,
-				self.space.global_transform(),
-				frame_info.proj_matrix,
-				None,
-				None,
-				None,
-			)
-		}
-	}
 }
 impl AspectIdentifier for CameraItem {
 	impl_aspect_for_camera_item_aspect_id! {}
@@ -193,15 +139,6 @@ impl CameraItemAcceptorAspect for CameraItemAcceptor {
 	}
 }
 
-pub fn update(token: &MainThreadToken) {
-	for camera in ITEM_TYPE_INFO_CAMERA.items.get_valid_contents() {
-		let ItemType::Camera(camera) = &camera.specialization else {
-			continue;
-		};
-		camera.update(token);
-	}
-}
-
 impl InterfaceAspect for Interface {
 	#[doc = "Create a camera item at a specific location"]
 	fn create_camera_item(

src/nodes/items/mod.rs

@@ -133,6 +133,7 @@ impl Drop for Item {
 	}
 }
 
+#[cfg_attr(not(feature = "wayland"), allow(dead_code))]
 pub enum ItemType {
 	Camera(Arc<CameraItem>),
 	Panel(Arc<dyn PanelItemTrait>),

src/nodes/items/panel.rs

@@ -102,6 +102,7 @@ pub struct PanelItem<B: Backend> {
 	pub backend: Box<B>,
 }
 impl<B: Backend> PanelItem<B> {
+	#[cfg_attr(not(feature = "wayland"), allow(dead_code))]
 	pub fn create(backend: Box<B>, pid: Option<i32>) -> (Arc<Node>, Arc<PanelItem<B>>) {
 		debug!(?pid, "Create panel item");
 

src/nodes/spatial/mod.rs

@@ -9,15 +9,85 @@ use crate::core::client::Client;
 use crate::core::error::Result;
 use crate::core::registry::Registry;
 use crate::nodes::{Node, OWNED_ASPECT_ALIAS_INFO};
+use bevy::prelude::Transform as BevyTransform;
+use bevy::prelude::*;
+use bevy::render::primitives::Aabb;
 use color_eyre::eyre::OptionExt;
 use glam::{Mat4, Quat, Vec3, vec3a};
 use mint::Vector3;
 use parking_lot::Mutex;
 use rustc_hash::FxHashMap;
 use std::fmt::Debug;
+use std::sync::atomic::Ordering;
 use std::sync::{Arc, OnceLock, Weak};
 use std::{f32, ptr};
-use stereokit_rust::maths::Bounds;
+
+pub struct SpatialNodePlugin;
+impl Plugin for SpatialNodePlugin {
+	fn build(&self, app: &mut App) {
+		app.add_systems(
+			PostUpdate,
+			update_spatial_nodes.before(TransformSystem::TransformPropagate),
+		);
+	}
+}
+
+fn update_spatial_nodes(
+	mut query: Query<(
+		&mut BevyTransform,
+		&SpatialNode,
+		Option<&ChildOf>,
+		&mut Visibility,
+	)>,
+	parent_query: Query<&GlobalTransform>,
+) {
+	query
+		.par_iter_mut()
+		.for_each(|(mut transform, spatial_node, child_of, mut vis)| {
+			let _span = debug_span!("updating spatial node").entered();
+			let Some(spatial) = spatial_node.0.upgrade() else {
+				return;
+			};
+			if spatial
+				.node()
+				.is_some_and(|v| !v.enabled.load(Ordering::Relaxed))
+			{
+				if !matches!(*vis, Visibility::Hidden) {
+					*vis = Visibility::Hidden;
+				}
+				return;
+			}
+			let mat4 =
+				debug_span!("getting global transform").in_scope(|| spatial.global_transform());
+			let (scale, _, _) = mat4.to_scale_rotation_translation();
+			match (*vis, scale == Vec3::ZERO) {
+				(Visibility::Inherited | Visibility::Visible, true) => {
+					*vis = Visibility::Hidden;
+				}
+				(Visibility::Hidden, false) => {
+					*vis = Visibility::Inherited;
+				}
+				_ => {}
+			}
+			match child_of {
+				Some(child_of) => {
+					let Ok(parent) = parent_query.get(child_of.0) else {
+						warn!("SpatialNode bevy Parent doesn't have global transform");
+						return;
+					};
+					*transform =
+						BevyTransform::from_matrix(parent.compute_matrix().inverse() * mat4);
+				}
+				None => {
+					*transform = BevyTransform::from_matrix(mat4);
+				}
+			}
+		});
+}
+
+#[derive(Clone, Component, Debug)]
+#[require(BevyTransform, Visibility)]
+pub struct SpatialNode(pub Weak<Spatial>);
 
 stardust_xr_server_codegen::codegen_spatial_protocol!();
 impl Transform {
@@ -58,7 +128,7 @@ pub struct Spatial {
 	transform: Mutex<Mat4>,
 	zone: Mutex<Weak<Zone>>,
 	children: Registry<Spatial>,
-	pub bounding_box_calc: OnceLock<fn(&Node) -> Bounds>,
+	pub bounding_box_calc: OnceLock<fn(&Node) -> Aabb>,
 }
 
 impl Spatial {
@@ -102,9 +172,9 @@ impl Spatial {
 	}
 
 	// the output bounds are probably way bigger than they need to be
-	pub fn get_bounding_box(&self) -> Bounds {
+	pub fn get_bounding_box(&self) -> Aabb {
 		let Some(node) = self.node() else {
-			return Bounds::default();
+			return Aabb::default();
 		};
 		let mut bounds = self
 			.bounding_box_calc
@@ -112,7 +182,15 @@ impl Spatial {
 			.map(|b| (b)(&node))
 			.unwrap_or_default();
 		for child in self.children.get_valid_contents() {
-			bounds.grown_box(child.get_bounding_box(), child.local_transform());
+			let mat = child.local_transform();
+			let child_aabb = child.get_bounding_box();
+			bounds = Aabb::enclosing([
+				bounds.min().into(),
+				bounds.max().into(),
+				mat.transform_point3(child_aabb.min().into()),
+				mat.transform_point3(child_aabb.max().into()),
+			])
+			.unwrap();
 		}
 		bounds
 	}
@@ -338,7 +416,7 @@ impl SpatialRefAspect for SpatialRef {
 
 		Ok(BoundingBox {
 			center: Vec3::from(bounds.center).into(),
-			size: Vec3::from(bounds.dimensions).into(),
+			size: Vec3::from(bounds.half_extents * 2.0).into(),
 		})
 	}
 
@@ -349,20 +427,17 @@ impl SpatialRefAspect for SpatialRef {
 	) -> Result<BoundingBox> {
 		let this_spatial = node.get_aspect::<Spatial>()?;
 		let relative_spatial = relative_to.get_aspect::<Spatial>()?;
-		let center = Spatial::space_to_space_matrix(Some(&this_spatial), Some(&relative_spatial))
-			.transform_point3([0.0; 3].into());
-		let mut bounds = Bounds {
-			center: center.into(),
-			dimensions: [0.0; 3].into(),
-		};
-		bounds.grown_box(
-			this_spatial.get_bounding_box(),
-			Spatial::space_to_space_matrix(Some(&this_spatial), Some(&relative_spatial)),
-		);
+		let mat = Spatial::space_to_space_matrix(Some(&this_spatial), Some(&relative_spatial));
+		let bb = this_spatial.get_bounding_box();
+		let bounds = Aabb::enclosing([
+			mat.transform_point3(bb.min().into()),
+			mat.transform_point3(bb.max().into()),
+		])
+		.unwrap();
 
 		Ok(BoundingBox {
 			center: Vec3::from(bounds.center).into(),
-			size: Vec3::from(bounds.dimensions).into(),
+			size: Vec3::from(bounds.half_extents * 2.0).into(),
 		})
 	}
 

src/objects/input/eye_pointer.rs

@@ -12,7 +12,6 @@ use glam::{Mat4, vec3};
 use serde::{Deserialize, Serialize};
 use stardust_xr::values::Datamap;
 use std::sync::Arc;
-use stereokit_rust::system::Input;
 
 #[derive(Default, Deserialize, Serialize)]
 pub struct EyeDatamap {
@@ -49,58 +48,4 @@ impl EyePointer {
 			pointer,
 		})
 	}
-	pub fn update(&self) {
-		let ray = Input::get_eyes();
-		self.spatial
-			.set_local_transform(Mat4::from_rotation_translation(
-				ray.orientation.into(),
-				ray.position.into(),
-			));
-		{
-			// Set pointer input datamap
-			*self.pointer.datamap.lock() = Datamap::from_typed(EyeDatamap { eye: 2 }).unwrap();
-		}
-
-		// send input to all the input handlers that are the closest to the ray as possible
-		let rx = INPUT_HANDLER_REGISTRY
-			.get_valid_contents()
-			.into_iter()
-			// filter out all the disabled handlers
-			.filter(|handler| {
-				let Some(node) = handler.spatial.node() else {
-					return false;
-				};
-				node.enabled()
-			})
-			// ray march to all the enabled handlers' fields
-			.map(|handler| {
-				let result = handler.field.ray_march(Ray {
-					origin: vec3(0.0, 0.0, 0.0),
-					direction: vec3(0.0, 0.0, -1.0),
-					space: self.spatial.clone(),
-				});
-				(vec![handler], result)
-			})
-			// make sure the field isn't at the pointer origin and that it's being hit
-			.filter(|(_, result)| result.deepest_point_distance > 0.01 && result.min_distance < 0.0)
-			// .inspect(|(_, result)| {
-			// 	dbg!(result);
-			// })
-			// now collect all handlers that are same distance if they're the closest
-			.reduce(|(mut handlers_a, result_a), (handlers_b, result_b)| {
-				if (result_a.deepest_point_distance - result_b.deepest_point_distance).abs() < 0.001
-				{
-					// distance is basically the same
-					handlers_a.extend(handlers_b);
-					(handlers_a, result_a)
-				} else if result_a.deepest_point_distance < result_b.deepest_point_distance {
-					(handlers_a, result_a)
-				} else {
-					(handlers_b, result_b)
-				}
-			})
-			.map(|(rx, _)| rx)
-			.unwrap_or_default();
-		self.pointer.set_handler_order(rx.iter());
-	}
 }

src/objects/input/mouse_pointer.rs

@@ -1,5 +1,6 @@
 use super::{CaptureManager, DistanceCalculator, get_sorted_handlers};
 use crate::{
+	DbusConnection, ObjectRegistryRes,
 	core::client::INTERNAL_CLIENT,
 	nodes::{
 		Node, OwnedNode,
@@ -9,6 +10,15 @@ use crate::{
 		spatial::Spatial,
 	},
 };
+use bevy::{
+	input::{
+		ButtonState,
+		keyboard::{KeyboardInput, NativeKey, NativeKeyCode},
+		mouse::{MouseMotion, MouseWheel},
+	},
+	prelude::*,
+	window::PrimaryWindow,
+};
 use color_eyre::eyre::Result;
 use glam::{Mat4, Vec3, vec3};
 use mint::Vector2;
@@ -19,12 +29,102 @@ use stardust_xr::{
 	values::Datamap,
 };
 use std::sync::Arc;
-use stereokit_rust::system::{Input, Key};
 use tokio::task::JoinSet;
 use tokio::time::{Duration, timeout};
 use xkbcommon_rs::{Context, Keymap, KeymapFormat, xkb_keymap::CompileFlags};
 use zbus::{Connection, names::OwnedInterfaceName};
 
+pub struct FlatscreenInputPlugin;
+
+impl Plugin for FlatscreenInputPlugin {
+	fn build(&self, app: &mut App) {
+		app.add_systems(Startup, setup);
+		// yes the input method will be delayed by one frame, its only for debugging anyways
+		app.add_systems(Update, update_pointer);
+	}
+}
+
+#[derive(Component)]
+#[require(Camera3d)]
+struct FlatscreenCam;
+
+fn setup(mut cmds: Commands) {
+	let Ok(pointer) =
+		MousePointer::new().inspect_err(|err| error!("unable to create mouse pointer: {err}"))
+	else {
+		return;
+	};
+	cmds.spawn((FlatscreenCam, Name::new("Flatscreen Camera")));
+	cmds.insert_resource(pointer);
+}
+
+fn update_pointer(
+	window: Single<(&Window), With<PrimaryWindow>>,
+	mut cam: Single<(&Camera, &GlobalTransform, &mut Transform), With<FlatscreenCam>>,
+	mut pointer: ResMut<MousePointer>,
+	connection: Res<DbusConnection>,
+	object_registry: Res<ObjectRegistryRes>,
+	mouse_buttons: Res<ButtonInput<MouseButton>>,
+	keyboard_buttons: Res<ButtonInput<KeyCode>>,
+	mut scroll: EventReader<MouseWheel>,
+	mut motion: EventReader<MouseMotion>,
+	mut keyboard_input_events: EventReader<KeyboardInput>,
+	time: Res<Time>,
+) {
+	let (cam, cam_transform, mut cam_local_transform) = cam.into_inner();
+	if keyboard_buttons.pressed(KeyCode::ShiftLeft) && mouse_buttons.pressed(MouseButton::Right) {
+		let (mut yaw, mut pitch, _) = cam_local_transform.rotation.to_euler(EulerRot::YXZ);
+
+		for e in motion.read() {
+			let scale = -0.003;
+			pitch += e.delta.y * scale;
+			yaw += e.delta.x * scale;
+		}
+
+		cam_local_transform.rotation = Quat::from_rotation_y(yaw) * Quat::from_rotation_x(pitch);
+
+		let mut move_vec = Vec3::ZERO;
+		move_vec.x += keyboard_buttons.pressed(KeyCode::KeyD) as u32 as f32;
+		move_vec.x -= keyboard_buttons.pressed(KeyCode::KeyA) as u32 as f32;
+		move_vec.z += keyboard_buttons.pressed(KeyCode::KeyS) as u32 as f32;
+		move_vec.z -= keyboard_buttons.pressed(KeyCode::KeyW) as u32 as f32;
+		move_vec.y += keyboard_buttons.pressed(KeyCode::KeyE) as u32 as f32;
+		move_vec.y -= keyboard_buttons.pressed(KeyCode::KeyQ) as u32 as f32;
+
+		let move_vec = cam_local_transform.rotation * move_vec.normalize_or_zero();
+		cam_local_transform.translation += move_vec * time.delta_secs() * 3.0;
+
+		return;
+	}
+	let Some(ray) = window
+		.cursor_position()
+		.and_then(|pos| get_viewport_pos(pos, cam))
+		.and_then(|pos| cam.viewport_to_world(cam_transform, pos).ok())
+	else {
+		return;
+	};
+	pointer.update(
+		&connection,
+		&object_registry,
+		ray,
+		&mouse_buttons,
+		&keyboard_buttons,
+		scroll,
+		keyboard_input_events,
+	);
+}
+
+fn get_viewport_pos(logical_pos: Vec2, cam: &Camera) -> Option<Vec2> {
+	if let Some(viewport_rect) = cam.logical_viewport_rect() {
+		if !viewport_rect.contains(logical_pos) {
+			return None;
+		}
+		Some(logical_pos - viewport_rect.min)
+	} else {
+		Some(logical_pos)
+	}
+}
+
 #[derive(Debug, Deserialize, Serialize)]
 struct MouseEvent {
 	select: f32,
@@ -59,7 +159,7 @@ trait KeyboardHandler {
 	async fn reset(&self) -> zbus::Result<()>;
 }
 
-#[allow(unused)]
+#[derive(Resource)]
 pub struct MousePointer {
 	node: OwnedNode,
 	keymap: DefaultKey,
@@ -95,35 +195,51 @@ impl MousePointer {
 			keymap,
 		})
 	}
-	pub fn update(&mut self, dbus_connection: &Connection, object_registry: &ObjectRegistry) {
-		let mouse = Input::get_mouse();
+	pub fn update(
+		&mut self,
+		dbus_connection: &Connection,
+		object_registry: &ObjectRegistry,
+		ray: Ray3d,
+		mouse_buttons: &ButtonInput<MouseButton>,
+		keyboard_buttons: &ButtonInput<KeyCode>,
+		mut scroll: EventReader<MouseWheel>,
+		mut keyboard_input_events: EventReader<KeyboardInput>,
+	) {
+		let mut discrete = Vec2::ZERO;
+		let mut continuous = Vec2::ZERO;
+		for e in scroll.read() {
+			match e.unit {
+				bevy::input::mouse::MouseScrollUnit::Line => {
+					discrete.x += e.x;
+					discrete.y += e.y;
+				}
+				bevy::input::mouse::MouseScrollUnit::Pixel => {
+					continuous.x += e.x;
+					continuous.y += e.y;
+				}
+			}
+		}
 
-		let ray = mouse.get_ray();
 		self.spatial.set_local_transform(
-			Mat4::look_to_rh(
-				Vec3::from(ray.position),
-				Vec3::from(ray.direction),
-				vec3(0.0, 1.0, 0.0),
-			)
-			.inverse(),
+			Mat4::look_to_rh(ray.origin, Vec3::from(ray.direction), Vec3::Y).inverse(),
 		);
 		{
 			// Set pointer input datamap
 			self.mouse_datamap = MouseEvent {
-				select: Input::key(Key::MouseLeft).is_active() as u32 as f32,
-				middle: Input::key(Key::MouseCenter).is_active() as u32 as f32,
-				context: Input::key(Key::MouseRight).is_active() as u32 as f32,
-				grab: (Input::key(Key::MouseRight).is_active()
-					|| (Input::key(Key::Backtick).is_active()
-						&& Input::key(Key::Shift).is_active())) as u32 as f32, // Was Mouse 5
-				scroll_continuous: [0.0, mouse.scroll_change / 120.0].into(),
-				scroll_discrete: [0.0, mouse.scroll_change / 120.0].into(),
+				select: mouse_buttons.pressed(MouseButton::Left) as u32 as f32,
+				middle: mouse_buttons.pressed(MouseButton::Middle) as u32 as f32,
+				context: mouse_buttons.pressed(MouseButton::Right) as u32 as f32,
+				grab: (mouse_buttons.pressed(MouseButton::Right)
+					|| (keyboard_buttons.pressed(KeyCode::Backquote)
+						&& keyboard_buttons.pressed(KeyCode::ShiftLeft))) as u32 as f32, // Was Mouse 5
+				scroll_continuous: [0.0, continuous.y / 120.0].into(),
+				scroll_discrete: [0.0, discrete.y / 120.0].into(),
 				raw_input_events: vec![],
 			};
 			*self.pointer.datamap.lock() = Datamap::from_typed(&self.mouse_datamap).unwrap();
 		}
 		self.target_pointer_input();
-		self.send_keyboard_input(dbus_connection, object_registry);
+		self.send_keyboard_input(dbus_connection, object_registry, keyboard_input_events);
 	}
 	fn target_pointer_input(&mut self) {
 		let distance_calculator: DistanceCalculator = |space, data, field| {
@@ -164,9 +280,13 @@ impl MousePointer {
 		&mut self,
 		dbus_connection: &Connection,
 		object_registry: &ObjectRegistry,
+		mut keyboard_input_events: EventReader<KeyboardInput>,
 	) {
 		let keyboard_handlers = object_registry.get_objects("org.stardustxr.XKBv1");
-
+		let events = keyboard_input_events
+			.read()
+			.filter_map(|e| Some((map_key(e.key_code)?, e.state)))
+			.collect::<Vec<_>>();
 		// Spawn async task to handle keyboard input
 		tokio::spawn({
 			let keyboard_handlers = keyboard_handlers.clone();
@@ -237,81 +357,75 @@ impl MousePointer {
 				let _ = keyboard_handler.keymap(keymap_id).await;
 
 				// Send key states
-				for i in 8_u32..254 {
-					let key = unsafe { std::mem::transmute::<u32, stereokit_rust::system::Key>(i) };
-					let Some(mapped_key) = map_key(key) else {
-						continue;
-					};
-					let input_state = Input::key(key);
-					if input_state.is_just_active() {
-						let _ = keyboard_handler.key_state(mapped_key + 8, true).await;
-					} else if input_state.is_just_inactive() {
-						let _ = keyboard_handler.key_state(mapped_key + 8, false).await;
-					}
+				for (key, state) in events.iter() {
+					let pressed = matches!(state, ButtonState::Pressed);
+					let _ = keyboard_handler.key_state(key + 8, pressed).await;
 				}
 			}
 		});
 	}
 }
 
-fn map_key(key: Key) -> Option<u32> {
+fn map_key(key: KeyCode) -> Option<u32> {
+	use KeyCode as Key;
 	match key {
+		Key::Unidentified(NativeKeyCode::Xkb(code)) => Some(code),
 		Key::Backspace => Some(input_event_codes::KEY_BACKSPACE!()),
 		Key::Tab => Some(input_event_codes::KEY_TAB!()),
-		Key::Return => Some(input_event_codes::KEY_ENTER!()),
-		Key::Shift => Some(input_event_codes::KEY_LEFTSHIFT!()),
-		Key::Ctrl => Some(input_event_codes::KEY_LEFTCTRL!()),
-		Key::Alt => Some(input_event_codes::KEY_LEFTALT!()),
+		Key::Enter => Some(input_event_codes::KEY_ENTER!()),
+		Key::ShiftLeft => Some(input_event_codes::KEY_LEFTSHIFT!()),
+		Key::ControlLeft => Some(input_event_codes::KEY_LEFTCTRL!()),
+		Key::AltLeft => Some(input_event_codes::KEY_LEFTALT!()),
 		Key::CapsLock => Some(input_event_codes::KEY_CAPSLOCK!()),
-		Key::Esc => Some(input_event_codes::KEY_ESC!()),
+		Key::Escape => Some(input_event_codes::KEY_ESC!()),
 		Key::Space => Some(input_event_codes::KEY_SPACE!()),
 		Key::End => Some(input_event_codes::KEY_END!()),
 		Key::Home => Some(input_event_codes::KEY_HOME!()),
-		Key::Left => Some(input_event_codes::KEY_LEFT!()),
-		Key::Right => Some(input_event_codes::KEY_RIGHT!()),
-		Key::Up => Some(input_event_codes::KEY_UP!()),
-		Key::Down => Some(input_event_codes::KEY_DOWN!()),
+		Key::ArrowLeft => Some(input_event_codes::KEY_LEFT!()),
+		Key::ArrowRight => Some(input_event_codes::KEY_RIGHT!()),
+		Key::ArrowUp => Some(input_event_codes::KEY_UP!()),
+		Key::ArrowDown => Some(input_event_codes::KEY_DOWN!()),
 		Key::PageUp => Some(input_event_codes::KEY_PAGEUP!()),
 		Key::PageDown => Some(input_event_codes::KEY_PAGEDOWN!()),
 		Key::PrintScreen => Some(input_event_codes::KEY_PRINT!()),
-		Key::KeyInsert => Some(input_event_codes::KEY_INSERT!()),
-		Key::Del => Some(input_event_codes::KEY_DELETE!()),
-		Key::Key0 => Some(input_event_codes::KEY_0!()),
-		Key::Key1 => Some(input_event_codes::KEY_1!()),
-		Key::Key2 => Some(input_event_codes::KEY_2!()),
-		Key::Key3 => Some(input_event_codes::KEY_3!()),
-		Key::Key4 => Some(input_event_codes::KEY_4!()),
-		Key::Key5 => Some(input_event_codes::KEY_5!()),
-		Key::Key6 => Some(input_event_codes::KEY_6!()),
-		Key::Key7 => Some(input_event_codes::KEY_7!()),
-		Key::Key8 => Some(input_event_codes::KEY_8!()),
-		Key::Key9 => Some(input_event_codes::KEY_9!()),
-		Key::A => Some(input_event_codes::KEY_A!()),
-		Key::B => Some(input_event_codes::KEY_B!()),
-		Key::C => Some(input_event_codes::KEY_C!()),
-		Key::D => Some(input_event_codes::KEY_D!()),
-		Key::E => Some(input_event_codes::KEY_E!()),
-		Key::F => Some(input_event_codes::KEY_F!()),
-		Key::G => Some(input_event_codes::KEY_G!()),
-		Key::H => Some(input_event_codes::KEY_H!()),
-		Key::I => Some(input_event_codes::KEY_I!()),
-		Key::J => Some(input_event_codes::KEY_J!()),
-		Key::K => Some(input_event_codes::KEY_K!()),
-		Key::L => Some(input_event_codes::KEY_L!()),
-		Key::M => Some(input_event_codes::KEY_M!()),
-		Key::N => Some(input_event_codes::KEY_N!()),
-		Key::O => Some(input_event_codes::KEY_O!()),
-		Key::P => Some(input_event_codes::KEY_P!()),
-		Key::Q => Some(input_event_codes::KEY_Q!()),
-		Key::R => Some(input_event_codes::KEY_R!()),
-		Key::S => Some(input_event_codes::KEY_S!()),
-		Key::T => Some(input_event_codes::KEY_T!()),
-		Key::U => Some(input_event_codes::KEY_U!()),
-		Key::V => Some(input_event_codes::KEY_V!()),
-		Key::W => Some(input_event_codes::KEY_W!()),
-		Key::X => Some(input_event_codes::KEY_X!()),
-		Key::Y => Some(input_event_codes::KEY_Y!()),
-		Key::Z => Some(input_event_codes::KEY_Z!()),
+		Key::Insert => Some(input_event_codes::KEY_INSERT!()),
+		Key::Delete => Some(input_event_codes::KEY_DELETE!()),
+		Key::Digit0 => Some(input_event_codes::KEY_0!()),
+		Key::Digit1 => Some(input_event_codes::KEY_1!()),
+		Key::Digit2 => Some(input_event_codes::KEY_2!()),
+		Key::Digit3 => Some(input_event_codes::KEY_3!()),
+		Key::Digit4 => Some(input_event_codes::KEY_4!()),
+		Key::Digit5 => Some(input_event_codes::KEY_5!()),
+		Key::Digit6 => Some(input_event_codes::KEY_6!()),
+		Key::Digit7 => Some(input_event_codes::KEY_7!()),
+		Key::Digit8 => Some(input_event_codes::KEY_8!()),
+		Key::Digit9 => Some(input_event_codes::KEY_9!()),
+		Key::KeyA => Some(input_event_codes::KEY_A!()),
+		Key::KeyB => Some(input_event_codes::KEY_B!()),
+		Key::KeyC => Some(input_event_codes::KEY_C!()),
+		Key::KeyD => Some(input_event_codes::KEY_D!()),
+		Key::KeyE => Some(input_event_codes::KEY_E!()),
+		Key::KeyF => Some(input_event_codes::KEY_F!()),
+		Key::KeyG => Some(input_event_codes::KEY_G!()),
+		Key::KeyH => Some(input_event_codes::KEY_H!()),
+		Key::KeyI => Some(input_event_codes::KEY_I!()),
+		Key::KeyJ => Some(input_event_codes::KEY_J!()),
+		Key::KeyK => Some(input_event_codes::KEY_K!()),
+		Key::KeyL => Some(input_event_codes::KEY_L!()),
+		Key::KeyM => Some(input_event_codes::KEY_M!()),
+		Key::KeyN => Some(input_event_codes::KEY_N!()),
+		Key::KeyO => Some(input_event_codes::KEY_O!()),
+		Key::KeyP => Some(input_event_codes::KEY_P!()),
+		Key::KeyQ => Some(input_event_codes::KEY_Q!()),
+		Key::KeyR => Some(input_event_codes::KEY_R!()),
+		Key::KeyS => Some(input_event_codes::KEY_S!()),
+		Key::KeyT => Some(input_event_codes::KEY_T!()),
+		Key::KeyU => Some(input_event_codes::KEY_U!()),
+		Key::KeyV => Some(input_event_codes::KEY_V!()),
+		Key::KeyW => Some(input_event_codes::KEY_W!()),
+		Key::KeyX => Some(input_event_codes::KEY_X!()),
+		Key::KeyY => Some(input_event_codes::KEY_Y!()),
+		Key::KeyZ => Some(input_event_codes::KEY_Z!()),
 		Key::Numpad0 => Some(input_event_codes::KEY_NUMERIC_0!()),
 		Key::Numpad1 => Some(input_event_codes::KEY_NUMERIC_1!()),
 		Key::Numpad2 => Some(input_event_codes::KEY_NUMERIC_2!()),
@@ -336,22 +450,22 @@ fn map_key(key: Key) -> Option<u32> {
 		Key::F12 => Some(input_event_codes::KEY_F12!()),
 		Key::Comma => Some(input_event_codes::KEY_COMMA!()),
 		Key::Period => Some(input_event_codes::KEY_DOT!()),
-		Key::SlashFwd => Some(input_event_codes::KEY_SLASH!()),
-		Key::SlashBack => Some(input_event_codes::KEY_BACKSLASH!()),
+		Key::Slash => Some(input_event_codes::KEY_SLASH!()),
+		Key::Backslash => Some(input_event_codes::KEY_BACKSLASH!()),
 		Key::Semicolon => Some(input_event_codes::KEY_SEMICOLON!()),
-		Key::Apostrophe => Some(input_event_codes::KEY_APOSTROPHE!()),
-		Key::BracketOpen => Some(input_event_codes::KEY_LEFTBRACE!()),
-		Key::BracketClose => Some(input_event_codes::KEY_RIGHTBRACE!()),
+		Key::Quote => Some(input_event_codes::KEY_APOSTROPHE!()),
+		Key::BracketLeft => Some(input_event_codes::KEY_LEFTBRACE!()),
+		Key::BracketRight => Some(input_event_codes::KEY_RIGHTBRACE!()),
 		Key::Minus => Some(input_event_codes::KEY_MINUS!()),
-		Key::Equals => Some(input_event_codes::KEY_EQUAL!()),
-		Key::Backtick => Some(input_event_codes::KEY_GRAVE!()),
-		Key::LCmd => Some(input_event_codes::KEY_LEFTMETA!()),
-		Key::RCmd => Some(input_event_codes::KEY_RIGHTMETA!()),
-		Key::Multiply => Some(input_event_codes::KEY_NUMERIC_STAR!()),
-		Key::Add => Some(input_event_codes::KEY_KPPLUS!()),
-		Key::Subtract => Some(input_event_codes::KEY_MINUS!()),
-		Key::Decimal => Some(input_event_codes::KEY_DOT!()),
-		Key::Divide => Some(input_event_codes::KEY_SLASH!()),
+		Key::Equal => Some(input_event_codes::KEY_EQUAL!()),
+		Key::Backquote => Some(input_event_codes::KEY_GRAVE!()),
+		Key::SuperLeft => Some(input_event_codes::KEY_LEFTMETA!()),
+		Key::SuperRight => Some(input_event_codes::KEY_RIGHTMETA!()),
+		Key::NumpadMultiply => Some(input_event_codes::KEY_NUMERIC_STAR!()),
+		Key::NumpadAdd => Some(input_event_codes::KEY_KPPLUS!()),
+		Key::NumpadSubtract => Some(input_event_codes::KEY_MINUS!()),
+		Key::NumpadDecimal => Some(input_event_codes::KEY_DOT!()),
+		Key::NumpadDivide => Some(input_event_codes::KEY_SLASH!()),
 		_ => None,
 	}
 }

src/objects/input/sk_controller.rs

@@ -1,27 +1,240 @@
 use super::{CaptureManager, get_sorted_handlers};
 use crate::{
+	DbusConnection, PreFrameWait,
 	core::client::INTERNAL_CLIENT,
 	nodes::{
 		Node, OwnedNode,
+		drawable::{
+			MaterialParameter,
+			model::{Model, ModelPart},
+		},
 		fields::{Field, FieldTrait},
 		input::{INPUT_HANDLER_REGISTRY, InputDataType, InputHandler, InputMethod, Tip},
 		spatial::Spatial,
 	},
 	objects::{ObjectHandle, SpatialRef, Tracked},
 };
-use color_eyre::eyre::Result;
-use glam::{Mat4, Vec2, Vec3};
-use serde::{Deserialize, Serialize};
-use stardust_xr::values::Datamap;
-use std::sync::Arc;
-use stereokit_rust::{
-	material::Material,
-	model::Model,
-	sk::MainThreadToken,
-	system::{Handed, Input},
-	util::Color128,
+use bevy::{asset::Handle, ecs::resource::Resource};
+use bevy::{math::Affine3, prelude::*};
+use bevy_mod_openxr::{
+	action_binding::{OxrSendActionBindings, OxrSuggestActionBinding},
+	helper_traits::{ToIsometry3d, ToVec2},
+	resources::{OxrFrameState, OxrInstance},
+	session::OxrSession,
 };
+use bevy_mod_xr::{
+	hands::HandSide,
+	session::{XrPreDestroySession, XrSessionCreated, XrSessionCreatedEvent},
+	spaces::{XrPrimaryReferenceSpace, XrReferenceSpace, XrSpace},
+};
+use color_eyre::eyre::Result;
+use glam::{Affine3A, Mat4, Vec2, Vec3};
+use openxr::{Action, ActiveActionSet, SpaceLocationFlags};
+use serde::{Deserialize, Serialize};
+use stardust_xr::values::{Datamap, ResourceID, color::Rgb};
+use std::{
+	borrow::Cow,
+	fs,
+	path::{Path, PathBuf},
+	str::FromStr,
+	sync::Arc,
+};
+use tracing::instrument;
 use zbus::Connection;
+pub struct ControllerPlugin;
+const CURSOR_MODEL_PATH: &str = "/tmp/stardust_server/models/cursor.glb";
+impl Plugin for ControllerPlugin {
+	fn build(&self, app: &mut App) {
+		let cursor = include_bytes!("cursor.glb");
+		fs::create_dir_all(
+			PathBuf::from_str(CURSOR_MODEL_PATH)
+				.unwrap()
+				.parent()
+				.unwrap(),
+		);
+		fs::write(CURSOR_MODEL_PATH, cursor).expect("can't write tmp cursor model file");
+		app.add_systems(OxrSendActionBindings, suggest_bindings.run_if(run_once));
+		app.add_systems(
+			PostUpdate,
+			create_spaces.run_if(on_event::<XrSessionCreatedEvent>),
+		);
+		app.add_systems(XrPreDestroySession, destroy_spaces);
+		app.add_systems(Startup, setup.run_if(resource_exists::<OxrInstance>));
+		app.add_systems(PreFrameWait, update);
+	}
+}
+
+// the api is just slightly nicer when using the bevy_mod_openxr solution okay?
+fn suggest_bindings(
+	instance: Res<OxrInstance>,
+	actions: Res<Actions>,
+	mut suggest: EventWriter<OxrSuggestActionBinding>,
+) {
+	let mut bind_all = |interaction_profile: &'static str,
+	                    bindings: &[(openxr::sys::Action, &[&'static str])]| {
+		for (action, bindings) in bindings {
+			suggest.write(OxrSuggestActionBinding {
+				action: *action,
+				interaction_profile: interaction_profile.into(),
+				bindings: bindings.iter().copied().map(Cow::Borrowed).collect(),
+			});
+		}
+	};
+	bind_all(
+		"/interaction_profiles/oculus/touch_controller",
+		&[
+			(
+				actions.trigger.as_raw(),
+				&[
+					"/user/hand/left/input/trigger/value",
+					"/user/hand/right/input/trigger/value",
+				],
+			),
+			(
+				actions.stick_click.as_raw(),
+				&[
+					"/user/hand/left/input/thumbstick/click",
+					"/user/hand/right/input/thumbstick/click",
+				],
+			),
+			(
+				actions.button.as_raw(),
+				&[
+					"/user/hand/left/input/x/click",
+					"/user/hand/left/input/y/click",
+					"/user/hand/right/input/a/click",
+					"/user/hand/right/input/b/click",
+				],
+			),
+			(
+				actions.grip.as_raw(),
+				&[
+					"/user/hand/left/input/squeeze/value",
+					"/user/hand/right/input/squeeze/value",
+				],
+			),
+			(
+				actions.stick.as_raw(),
+				&[
+					"/user/hand/left/input/thumbstick",
+					"/user/hand/right/input/thumbstick",
+				],
+			),
+			(
+				actions.space.as_raw(),
+				&[
+					"/user/hand/left/input/aim/pose",
+					"/user/hand/right/input/aim/pose",
+				],
+			),
+		],
+	);
+	bind_all(
+		"/interaction_profiles/khr/simple_controller",
+		&[(
+			actions.space.as_raw(),
+			&[
+				"/user/hand/left/input/aim/pose",
+				"/user/hand/right/input/aim/pose",
+			],
+		)],
+	);
+}
+
+fn update(
+	mut controllers: ResMut<Controllers>,
+	actions: Res<Actions>,
+	session: Option<Res<OxrSession>>,
+	ref_space: Option<Res<XrPrimaryReferenceSpace>>,
+	state: Option<Res<OxrFrameState>>,
+) {
+	let (Some(session), Some(state), Some(ref_space)) = (session, state, ref_space) else {
+		controllers.left.set_enabled(false);
+		controllers.right.set_enabled(false);
+		return;
+	};
+	debug_span!("sync actions").in_scope(|| {
+		session
+			.sync_actions(&[ActiveActionSet::new(&actions.set)])
+			.unwrap();
+	});
+	let time = state.predicted_display_time;
+	// stupid bevy gltf loading issue (rotated 180 degrees on the y axis)
+	controllers
+		.left
+		.update(&session, &actions, time, ref_space.0);
+	controllers
+		.right
+		.update(&session, &actions, time, ref_space.0);
+}
+
+fn create_spaces(
+	session: Res<OxrSession>,
+	mut controllers: ResMut<Controllers>,
+	actions: Res<Actions>,
+) {
+	// if we ever need more actions than just these we should fully swith to the
+	// bevy_mod_openxr provided stuff
+	session.attach_action_sets(&[&actions.set]);
+	session
+		.sync_actions(&[ActiveActionSet::new(&actions.set)])
+		.unwrap();
+
+	let instance = session.instance();
+	let left = instance.string_to_path("/user/hand/left").unwrap();
+	let right = instance.string_to_path("/user/hand/right").unwrap();
+	let left = session
+		.create_action_space(&actions.space, left, Isometry3d::IDENTITY)
+		.unwrap();
+	let right = session
+		.create_action_space(&actions.space, right, Isometry3d::IDENTITY)
+		.unwrap();
+	controllers.left.space = Some(left);
+	controllers.right.space = Some(right);
+}
+
+fn destroy_spaces(session: Res<OxrSession>, mut controllers: ResMut<Controllers>) {
+	if let Some(space) = controllers.left.space.take() {
+		session.destroy_space(space);
+	}
+	if let Some(space) = controllers.right.space.take() {
+		session.destroy_space(space);
+	}
+}
+
+fn setup(instance: Res<OxrInstance>, connection: Res<DbusConnection>, mut cmds: Commands) {
+	tokio::task::spawn({
+		let connection = connection.clone();
+		async move {
+			connection
+				.request_name("org.stardustxr.Controllers")
+				.await
+				.unwrap();
+		}
+	});
+	let set = instance
+		.create_action_set("input_method_actions", "Input Method Action Source", 0)
+		.unwrap();
+	let paths = &[
+		instance.string_to_path("/user/hand/left").unwrap(),
+		instance.string_to_path("/user/hand/right").unwrap(),
+	];
+	let actions = Actions {
+		trigger: set.create_action("trigger", "Select", paths).unwrap(),
+		stick_click: set.create_action("stick_click", "Middle", paths).unwrap(),
+		button: set.create_action("face_button", "Context", paths).unwrap(),
+		grip: set.create_action("grip", "Grab", paths).unwrap(),
+		stick: set.create_action("stick", "Scroll", paths).unwrap(),
+		space: set.create_action("pose", "Location", paths).unwrap(),
+		set,
+	};
+	let controllers = Controllers {
+		left: SkController::new(&connection, HandSide::Left).unwrap(),
+		right: SkController::new(&connection, HandSide::Right).unwrap(),
+	};
+	cmds.insert_resource(controllers);
+	cmds.insert_resource(actions);
+}
 
 #[derive(Default, Debug, Deserialize, Serialize)]
 struct ControllerDatamap {
@@ -31,58 +244,69 @@ struct ControllerDatamap {
 	grab: f32,
 	scroll: Vec2,
 }
+#[derive(Resource)]
+struct Actions {
+	set: openxr::ActionSet,
+	trigger: openxr::Action<f32>,
+	stick_click: openxr::Action<f32>,
+	button: openxr::Action<f32>,
+	grip: openxr::Action<f32>,
+	space: openxr::Action<openxr::Posef>,
+	stick: openxr::Action<openxr::Vector2f>,
+}
+#[derive(Resource)]
+struct Controllers {
+	left: SkController,
+	right: SkController,
+}
 
 pub struct SkController {
 	object_handle: ObjectHandle<SpatialRef>,
 	input: Arc<InputMethod>,
-	handed: Handed,
-	model: Model,
-	material: Material,
+	side: HandSide,
+	model: Arc<Model>,
+	model_part: Arc<ModelPart>,
 	capture_manager: CaptureManager,
 	datamap: ControllerDatamap,
 	tracked: ObjectHandle<Tracked>,
+	space: Option<XrSpace>,
 }
 impl SkController {
-	pub fn new(connection: &Connection, handed: Handed) -> Result<Self> {
-		Input::set_controller_model(handed, Some(Model::new()));
+	fn new(connection: &Connection, side: HandSide) -> Result<Self> {
 		let path = "/org/stardustxr/Controller/".to_string()
-			+ match handed {
-				Handed::Left => "left",
-				_ => "right",
+			+ match side {
+				HandSide::Left => "left",
+				HandSide::Right => "right",
 			};
 		let (spatial, object_handle) = SpatialRef::create(connection, &path);
 		let tracked = Tracked::new(connection, &path);
-		let model = Model::copy(&Model::from_memory(
-			"cursor.glb",
-			include_bytes!("cursor.glb"),
-			None,
-		)?);
-		let model_nodes = model.get_nodes();
-		let mut model_node = model_nodes.visuals().next().unwrap();
-		let material = Material::copy(&model_node.get_material().unwrap());
-		model_node.material(&material);
 		let tip = InputDataType::Tip(Tip::default());
+		let node = spatial.node().unwrap();
+		node.set_enabled(false);
+		let model = Model::add_to(&node, ResourceID::Direct(CURSOR_MODEL_PATH.into())).unwrap();
+		let model_part = model.get_model_part("Cursor".to_string()).unwrap();
 		let input = InputMethod::add_to(
-			&spatial.node().unwrap(),
+			&node,
 			tip,
 			Datamap::from_typed(ControllerDatamap::default())?,
 		)?;
 		Ok(SkController {
 			object_handle,
 			input,
-			handed,
+			side,
 			model,
-			material,
+			model_part,
 			capture_manager: CaptureManager::default(),
 			datamap: Default::default(),
 			tracked,
+			space: None,
 		})
 	}
-	pub fn update(&mut self, token: &MainThreadToken) {
-		let controller = Input::controller(self.handed);
-		let input_node = self.input.spatial.node().unwrap();
-		input_node.set_enabled(controller.tracked.is_active());
-		let enabled = input_node.enabled();
+	#[instrument(level = "debug", skip(self))]
+	pub fn set_enabled(&self, enabled: bool) {
+		if let Some(node) = self.input.spatial.node() {
+			node.set_enabled(enabled);
+		}
 		tokio::spawn({
 			// this is suboptimal since it probably allocates a fresh string every frame
 			let handle = self.tracked.clone();
@@ -90,34 +314,85 @@ impl SkController {
 				handle.set_tracked(enabled).await;
 			}
 		});
+	}
+	fn update(
+		&mut self,
+		session: &OxrSession,
+		actions: &Actions,
+		time: openxr::Time,
+		ref_space: XrReferenceSpace,
+	) {
+		let Some(space) = self.space.as_ref() else {
+			return;
+		};
+		let _span = debug_span!("locate space").entered();
+		let Ok(location) = session
+			.locate_space(space, &ref_space, time)
+			.inspect_err(|err| error!("error while locating controller space: {err}"))
+		else {
+			return;
+		};
+		let enabled = location.location_flags.contains(
+			SpaceLocationFlags::POSITION_VALID
+				| SpaceLocationFlags::POSITION_TRACKED
+				| SpaceLocationFlags::ORIENTATION_VALID
+				| SpaceLocationFlags::ORIENTATION_TRACKED,
+		);
+		drop(_span);
+		self.set_enabled(enabled);
 		if enabled {
-			let world_transform = Mat4::from_rotation_translation(
-				controller.aim.orientation.into(),
-				controller.aim.position.into(),
+			let world_transform = Mat4::from(Affine3A::from(location.pose.to_xr_pose()));
+			self.model_part
+				.set_material_parameter("roughness".to_string(), MaterialParameter::Float(1.0));
+			self.model_part.set_material_parameter(
+				"color".to_string(),
+				MaterialParameter::Color(stardust_xr::values::Color::new(
+					if self.capture_manager.capture.upgrade().is_none() {
+						Rgb::new(1.0, 1.0, 1.0)
+					} else {
+						Rgb::new(0.0, 1.0, 0.75)
+					},
+					1.0,
+				)),
 			);
-			self.material
-				.color_tint(if self.capture_manager.capture.upgrade().is_none() {
-					Color128::new_rgb(1.0, 1.0, 1.0)
-				} else {
-					Color128::new_rgb(0.0, 1.0, 0.75)
-				});
-			self.model.draw(
-				token,
-				world_transform * Mat4::from_scale(Vec3::ONE * 0.02),
-				None,
-				None,
-			);
-			self.input.spatial.set_local_transform(world_transform);
+			self.input
+				.spatial
+				.set_local_transform(world_transform * Mat4::from_scale(Vec3::splat(0.02)));
 		}
-
+		let path = session
+			.instance()
+			.string_to_path(match self.side {
+				HandSide::Left => "/user/hand/left",
+				HandSide::Right => "/user/hand/right",
+			})
+			.unwrap();
+		if let Ok(path) = session.current_interaction_profile(path) {
+			if session.instance().path_to_string(path).unwrap()
+				== "/interaction_profiles/khr/simple_controller"
+			{
+				self.set_enabled(false);
+			}
+		}
+		fn get<T: openxr::ActionInput + Default>(
+			session: &OxrSession,
+			path: openxr::Path,
+			action: &Action<T>,
+		) -> T {
+			action
+				.state(session, path)
+				.map(|v| v.current_state)
+				.unwrap_or_default()
+		}
+		let _span = debug_span!("apply datamap").entered();
 		self.datamap = ControllerDatamap {
-			select: controller.trigger,
-			middle: controller.stick_click.is_active() as u32 as f32,
-			context: controller.is_x2_pressed() as u32 as f32,
-			grab: controller.grip,
-			scroll: controller.stick.into(),
+			select: get(session, path, &actions.trigger),
+			middle: get(session, path, &actions.stick_click) as u32 as f32,
+			context: get(session, path, &actions.button) as u32 as f32,
+			grab: get(session, path, &actions.grip),
+			scroll: get(session, path, &actions.stick).to_vec2(),
 		};
 		*self.input.datamap.lock() = Datamap::from_typed(&self.datamap).unwrap();
+		drop(_span);
 
 		let distance_calculator = |space: &Arc<Spatial>, _data: &InputDataType, field: &Field| {
 			Some(field.distance(space, [0.0; 3].into()).abs())

src/objects/input/sk_hand.rs

@@ -1,35 +1,173 @@
 use crate::core::client::INTERNAL_CLIENT;
 use crate::nodes::OwnedNode;
 use crate::nodes::fields::{Field, FieldTrait};
-use crate::nodes::input::{INPUT_HANDLER_REGISTRY, InputDataType, InputHandler};
+use crate::nodes::input::{Finger, INPUT_HANDLER_REGISTRY, InputDataType, InputHandler, Thumb};
 use crate::nodes::{
 	Node,
 	input::{Hand, InputMethod, Joint},
 	spatial::Spatial,
 };
 use crate::objects::{ObjectHandle, SpatialRef, Tracked};
+use crate::{BevyMaterial, DbusConnection, ObjectRegistryRes, PreFrameWait};
+use bevy::prelude::Transform as BevyTransform;
+use bevy::prelude::*;
+use bevy_mod_openxr::helper_traits::{ToQuat, ToVec3};
+use bevy_mod_openxr::resources::OxrFrameState;
+use bevy_mod_openxr::session::OxrSession;
+use bevy_mod_xr::hands::{HandBone, HandSide, XrHandBoneEntities, XrHandBoneRadius};
+use bevy_mod_xr::session::{XrPreDestroySession, XrSessionCreated};
+use bevy_mod_xr::spaces::{XrPrimaryReferenceSpace, XrSpaceLocationFlags};
+use bevy_sk::hand::GRADIENT_TEXTURE_HANDLE;
 use color_eyre::eyre::Result;
 use glam::{Mat4, Quat, Vec3};
+use openxr::{HandJointLocation, SpaceLocationFlags};
 use serde::{Deserialize, Serialize};
 use stardust_xr::values::Datamap;
 use std::sync::Arc;
-use stereokit_rust::material::Material;
-use stereokit_rust::sk::{DisplayMode, MainThreadToken, Sk};
-use stereokit_rust::system::{HandJoint, HandSource, Handed, Input, LinePoint, Lines};
-use stereokit_rust::util::Color128;
 use zbus::Connection;
 
 use super::{CaptureManager, get_sorted_handlers};
 
-fn convert_joint(joint: HandJoint) -> Joint {
+pub struct HandPlugin;
+impl Plugin for HandPlugin {
+	fn build(&self, app: &mut App) {
+		app.add_systems(PreFrameWait, update_hands);
+		app.add_systems(XrSessionCreated, create_trackers);
+		app.add_systems(XrPreDestroySession, destroy_trackers);
+		app.add_systems(PostUpdate, update_hand_material);
+		app.add_systems(Startup, setup);
+	}
+}
+fn update_hands(
+	mut hands: ResMut<Hands>,
+	session: Option<Res<OxrSession>>,
+	state: Option<Res<OxrFrameState>>,
+	ref_space: Option<Res<XrPrimaryReferenceSpace>>,
+	mut materials: ResMut<Assets<BevyMaterial>>,
+	mut joint_query: Query<(
+		&mut BevyTransform,
+		&mut XrSpaceLocationFlags,
+		&mut XrHandBoneRadius,
+	)>,
+	joints_query: Query<&XrHandBoneEntities>,
+) {
+	let (Some(session), Some(state), Some(ref_space)) = (session, state, ref_space) else {
+		tokio::task::spawn({
+			let left = hands.left.tracked.clone();
+			let right = hands.right.tracked.clone();
+			async move {
+				left.set_tracked(false);
+				right.set_tracked(false);
+			}
+		});
+		return;
+	};
+	let get_joints = |hand: &mut SkHand| -> Option<openxr::HandJointLocations> {
+		let Some(tracker) = hand.tracker.as_ref() else {
+			hand.input.spatial.node().unwrap().set_enabled(false);
+			let handle = hand.tracked.clone();
+			tokio::task::spawn(async move {
+				handle.set_tracked(false);
+			});
+			return None;
+		};
+		// this won't be correct with pipelined rendering
+		session
+			.locate_hand_joints(tracker, &ref_space, state.predicted_display_time)
+			.inspect_err(|err| error!("Error while locating hand joints"))
+			.ok()
+			.flatten()
+	};
+	let joints_left = get_joints(&mut hands.left);
+	let joints_right = get_joints(&mut hands.right);
+	hands.left.update(joints_left.as_ref(), &mut materials);
+	hands.right.update(joints_right.as_ref(), &mut materials);
+}
+
+fn pinch_between(joint_1: &Joint, joint_2: &Joint) -> f32 {
+	const PINCH_MAX: f32 = 0.11;
+	const PINCH_ACTIVACTION_DISTANCE: f32 = 0.01;
+	let combined_radius = joint_1.radius + joint_2.radius;
+	let pinch_dist =
+		Vec3::from(joint_1.position).distance(Vec3::from(joint_2.position)) - combined_radius;
+	(1.0 - ((pinch_dist - PINCH_ACTIVACTION_DISTANCE) / (PINCH_MAX - PINCH_ACTIVACTION_DISTANCE)))
+		.clamp(0.0, 1.0)
+}
+
+fn create_trackers(session: Res<OxrSession>, mut hands: ResMut<Hands>) {
+	hands.left.tracker = session
+		.create_hand_tracker(openxr::HandEXT::LEFT)
+		.inspect_err(|err| error!("failed to create left hand tracker"))
+		.ok();
+	hands.right.tracker = session
+		.create_hand_tracker(openxr::HandEXT::RIGHT)
+		.inspect_err(|err| error!("failed to create right hand tracker"))
+		.ok();
+}
+fn destroy_trackers(mut hands: ResMut<Hands>) {
+	hands.left.tracker.take();
+	hands.right.tracker.take();
+}
+#[derive(Component)]
+struct CorrectHandMaterial;
+fn update_hand_material(
+	query: Query<
+		(Entity, &HandSide),
+		(
+			With<XrHandBoneEntities>,
+			With<MeshMaterial3d<BevyMaterial>>,
+			Without<CorrectHandMaterial>,
+		),
+	>,
+	mut cmds: Commands,
+	hands: Res<Hands>,
+) {
+	for (entity, side) in &query {
+		let handle = match side {
+			HandSide::Left => hands.left.material.clone(),
+			HandSide::Right => hands.right.material.clone(),
+		};
+		cmds.entity(entity)
+			.insert(MeshMaterial3d(handle))
+			.insert(CorrectHandMaterial);
+	}
+}
+
+fn setup(
+	connection: Res<DbusConnection>,
+	mut cmds: Commands,
+	mut materials: ResMut<Assets<BevyMaterial>>,
+) {
+	tokio::task::spawn({
+		let connection = connection.clone();
+		async move {
+			connection
+				.request_name("org.stardustxr.Hands")
+				.await
+				.unwrap();
+		}
+	});
+	cmds.insert_resource(Hands {
+		left: SkHand::new(&connection, HandSide::Left, &mut materials).unwrap(),
+		right: SkHand::new(&connection, HandSide::Right, &mut materials).unwrap(),
+	});
+}
+
+fn convert_joint(joint: HandJointLocation) -> Joint {
 	Joint {
-		position: Vec3::from(joint.position).into(),
-		rotation: Quat::from(joint.orientation).into(),
+		position: joint.pose.position.to_vec3().into(),
+		rotation: joint.pose.orientation.to_quat().into(),
 		radius: joint.radius,
 		distance: 0.0,
 	}
 }
 
+#[derive(Resource)]
+struct Hands {
+	left: SkHand,
+	right: SkHand,
+}
+
 #[derive(Default, Deserialize, Serialize)]
 struct HandDatamap {
 	pinch_strength: f32,
@@ -40,117 +178,163 @@ pub struct SkHand {
 	_node: OwnedNode,
 	palm_spatial: Arc<Spatial>,
 	palm_object: ObjectHandle<SpatialRef>,
-	handed: Handed,
+	side: HandSide,
 	input: Arc<InputMethod>,
 	capture_manager: CaptureManager,
 	datamap: HandDatamap,
 	tracked: ObjectHandle<Tracked>,
+	tracker: Option<openxr::HandTracker>,
+	captured: bool,
+	material: Handle<BevyMaterial>,
 }
 impl SkHand {
-	pub fn new(connection: &Connection, handed: Handed) -> Result<Self> {
+	pub fn new(
+		connection: &Connection,
+		side: HandSide,
+		materials: &mut Assets<BevyMaterial>,
+	) -> Result<Self> {
 		let (palm_spatial, palm_object) = SpatialRef::create(
 			connection,
 			&("/org/stardustxr/Hand/".to_string()
-				+ match handed {
-					Handed::Left => "left",
-					_ => "right",
+				+ match side {
+					HandSide::Left => "left",
+					HandSide::Right => "right",
 				} + "/palm"),
 		);
 		let tracked = Tracked::new(
 			connection,
 			&("/org/stardustxr/Hand/".to_string()
-				+ match handed {
-					Handed::Left => "left",
-					_ => "right",
+				+ match side {
+					HandSide::Left => "left",
+					HandSide::Right => "right",
 				}),
 		);
-		let _node = Node::generate(&INTERNAL_CLIENT, false).add_to_scenegraph_owned()?;
-		Spatial::add_to(&_node.0, None, Mat4::IDENTITY, false);
+		let node = Node::generate(&INTERNAL_CLIENT, false).add_to_scenegraph_owned()?;
+		Spatial::add_to(&node.0, None, Mat4::IDENTITY, false);
 		let hand = InputDataType::Hand(Hand {
-			right: handed == Handed::Right,
+			right: matches!(side, HandSide::Right),
 			..Default::default()
 		});
 		let datamap = Datamap::from_typed(HandDatamap::default())?;
-		let input = InputMethod::add_to(&_node.0, hand, datamap)?;
-		Input::hand_visible(handed, true);
+		let input = InputMethod::add_to(&node.0, hand, datamap)?;
 
+		let material = materials.add(BevyMaterial {
+			base_color: Srgba::new(1.0, 1.0, 1.0, 1.0).into(),
+			alpha_mode: AlphaMode::Blend,
+			base_color_texture: Some(GRADIENT_TEXTURE_HANDLE),
+			perceptual_roughness: 1.0,
+			..default()
+		});
 		Ok(SkHand {
-			_node,
+			_node: node,
 			palm_spatial,
 			palm_object,
-			handed,
+			side,
 			input,
 			tracked,
 			capture_manager: CaptureManager::default(),
 			datamap: Default::default(),
+			tracker: None,
+			material,
+			captured: false,
 		})
 	}
-	pub fn update(&mut self, sk: &Sk, token: &MainThreadToken, material: &mut Material) {
-		let sk_hand = Input::hand(self.handed);
-		let real_hand = Input::hand_source(self.handed) as u32 == HandSource::Articulated as u32;
-		if let InputDataType::Hand(hand) = &mut *self.input.data.lock() {
-			let input_node = self.input.spatial.node().unwrap();
-			input_node.set_enabled(
-				(real_hand || sk.get_active_display_mode() == DisplayMode::Flatscreen)
-					&& sk_hand.tracked.is_active(),
-			);
-			let enabled = input_node.enabled();
-			tokio::spawn({
-				// this is suboptimal since it probably allocates a fresh string every frame
-				let handle = self.tracked.clone();
-				async move {
-					handle.set_tracked(enabled).await;
-				}
-			});
-			if enabled {
-				hand.thumb.tip = convert_joint(sk_hand.fingers[0][4]);
-				hand.thumb.distal = convert_joint(sk_hand.fingers[0][3]);
-				hand.thumb.proximal = convert_joint(sk_hand.fingers[0][2]);
-				hand.thumb.metacarpal = convert_joint(sk_hand.fingers[0][1]);
-
-				for (finger, mut sk_finger) in [
-					(&mut hand.index, sk_hand.fingers[1]),
-					(&mut hand.middle, sk_hand.fingers[2]),
-					(&mut hand.ring, sk_hand.fingers[3]),
-					(&mut hand.little, sk_hand.fingers[4]),
-				] {
-					sk_finger[4].radius = 0.0;
-					finger.tip = convert_joint(sk_finger[4]);
-					finger.distal = convert_joint(sk_finger[3]);
-					finger.intermediate = convert_joint(sk_finger[2]);
-					finger.proximal = convert_joint(sk_finger[1]);
-					finger.metacarpal = convert_joint(sk_finger[0]);
-				}
-
-				hand.palm.position = Vec3::from(sk_hand.palm.position).into();
-				hand.palm.rotation = Quat::from(sk_hand.palm.orientation).into();
-				hand.palm.radius =
-					(sk_hand.fingers[2][0].radius + sk_hand.fingers[2][1].radius) * 0.5;
-
-				self.palm_spatial
-					.set_local_transform(Mat4::from_rotation_translation(
-						hand.palm.rotation.into(),
-						hand.palm.position.into(),
-					));
-
-				hand.wrist.position = Vec3::from(sk_hand.wrist.position).into();
-				hand.wrist.rotation = Quat::from(sk_hand.wrist.orientation).into();
-				hand.wrist.radius =
-					(sk_hand.fingers[0][0].radius + sk_hand.fingers[4][0].radius) * 0.5;
-
-				hand.elbow = None;
-
-				let hand_color = if self.capture_manager.capture.upgrade().is_none() {
-					Color128::new_rgb(1.0, 1.0, 1.0)
-				} else {
-					Color128::new_rgb(0.0, 1.0, 0.75)
-				};
-				material.color_tint(hand_color);
-			}
+	pub fn set_enabled(&self, enabled: bool) {
+		if let Some(node) = self.input.spatial.node() {
+			node.set_enabled(enabled);
+		}
+		tokio::spawn({
+			// this is suboptimal since it probably allocates a fresh string every frame
+			let handle = self.tracked.clone();
+			async move {
+				handle.set_tracked(enabled).await;
+			}
+		});
+	}
+	fn update(
+		&mut self,
+		joints: Option<&openxr::HandJointLocations>,
+		materials: &mut ResMut<Assets<BevyMaterial>>,
+	) {
+		// TODO: use the hand data source ext
+		let real_hand = true;
+		let input_node = self.input.spatial.node().unwrap();
+		let is_tracked = real_hand
+			&& joints.is_some_and(|v| {
+				v.iter().all(|v| {
+					v.location_flags.contains(
+						SpaceLocationFlags::POSITION_VALID | SpaceLocationFlags::POSITION_TRACKED,
+					) || v.location_flags.contains(
+						SpaceLocationFlags::ORIENTATION_VALID
+							| SpaceLocationFlags::ORIENTATION_TRACKED,
+					)
+				})
+			});
+		self.set_enabled(is_tracked);
+		if is_tracked {
+			// cannot ever crash, is_tracked is only true of joints is some
+			let joints = joints.unwrap();
+			let new_hand = Hand {
+				right: matches!(self.side, HandSide::Right),
+				thumb: Thumb {
+					tip: convert_joint(joints[HandBone::ThumbTip as usize]),
+					distal: convert_joint(joints[HandBone::ThumbDistal as usize]),
+					proximal: convert_joint(joints[HandBone::ThumbProximal as usize]),
+					metacarpal: convert_joint(joints[HandBone::ThumbMetacarpal as usize]),
+				},
+				index: Finger {
+					tip: convert_joint(joints[HandBone::IndexTip as usize]),
+					distal: convert_joint(joints[HandBone::IndexDistal as usize]),
+					intermediate: convert_joint(joints[HandBone::IndexIntermediate as usize]),
+					proximal: convert_joint(joints[HandBone::IndexProximal as usize]),
+					metacarpal: convert_joint(joints[HandBone::IndexMetacarpal as usize]),
+				},
+				middle: Finger {
+					tip: convert_joint(joints[HandBone::MiddleTip as usize]),
+					distal: convert_joint(joints[HandBone::MiddleDistal as usize]),
+					intermediate: convert_joint(joints[HandBone::MiddleIntermediate as usize]),
+					proximal: convert_joint(joints[HandBone::MiddleProximal as usize]),
+					metacarpal: convert_joint(joints[HandBone::MiddleMetacarpal as usize]),
+				},
+				ring: Finger {
+					tip: convert_joint(joints[HandBone::RingTip as usize]),
+					distal: convert_joint(joints[HandBone::RingDistal as usize]),
+					intermediate: convert_joint(joints[HandBone::RingIntermediate as usize]),
+					proximal: convert_joint(joints[HandBone::RingProximal as usize]),
+					metacarpal: convert_joint(joints[HandBone::RingMetacarpal as usize]),
+				},
+				little: Finger {
+					tip: convert_joint(joints[HandBone::LittleTip as usize]),
+					distal: convert_joint(joints[HandBone::LittleDistal as usize]),
+					intermediate: convert_joint(joints[HandBone::LittleIntermediate as usize]),
+					proximal: convert_joint(joints[HandBone::LittleProximal as usize]),
+					metacarpal: convert_joint(joints[HandBone::LittleMetacarpal as usize]),
+				},
+				palm: convert_joint(joints[HandBone::Palm as usize]),
+				wrist: convert_joint(joints[HandBone::Wrist as usize]),
+				elbow: None,
+			};
+			self.palm_spatial
+				.set_local_transform(Mat4::from_rotation_translation(
+					new_hand.palm.rotation.into(),
+					new_hand.palm.position.into(),
+				));
+
+			self.datamap.pinch_strength = pinch_between(&new_hand.thumb.tip, &new_hand.index.tip);
+			// this is how stereokit calculates grab
+			self.datamap.grab_strength =
+				pinch_between(&new_hand.ring.tip, &new_hand.ring.metacarpal);
+
+			*self.input.data.lock() = InputDataType::Hand(new_hand);
+			*self.input.datamap.lock() = Datamap::from_typed(&self.datamap).unwrap();
+			let captured = self.capture_manager.capture.upgrade().is_some();
+			if captured && !self.captured {
+				materials.get_mut(&self.material).unwrap().base_color = Srgba::rgb(0., 1., 0.75).into();
+			} else if self.captured && !captured {
+				materials.get_mut(&self.material).unwrap().base_color = Srgba::rgb(1., 1.0, 1.0).into();
+			}
+			self.captured = captured;
 		}
-		self.datamap.pinch_strength = sk_hand.pinch_activation;
-		self.datamap.grab_strength = sk_hand.grip_activation;
-		*self.input.datamap.lock() = Datamap::from_typed(&self.datamap).unwrap();
 
 		let distance_calculator = |space: &Arc<Spatial>, data: &InputDataType, field: &Field| {
 			let InputDataType::Hand(hand) = data else {
@@ -183,16 +367,3 @@ impl SkHand {
 			.set_handler_order(sorted_handlers.iter().map(|(handler, _)| handler));
 	}
 }
-impl Drop for SkHand {
-	fn drop(&mut self) {
-		Input::hand_visible(self.handed, false);
-	}
-}
-
-fn joint_to_line_point(joint: &Joint, color: Color128) -> LinePoint {
-	LinePoint {
-		pt: Vec3::from(joint.position).into(),
-		thickness: joint.radius * 2.0,
-		color: color.into(),
-	}
-}

src/objects/mod.rs

@@ -13,192 +13,18 @@ use input::{
 	eye_pointer::EyePointer, mouse_pointer::MousePointer, sk_controller::SkController,
 	sk_hand::SkHand,
 };
+use parking_lot::RwLock;
 use play_space::PlaySpaceBounds;
 use stardust_xr::schemas::dbus::object_registry::ObjectRegistry;
 use std::{
 	marker::PhantomData,
 	sync::{Arc, atomic::Ordering},
 };
-use stereokit_rust::{
-	material::Material,
-	sk::{DisplayMode, MainThreadToken, Sk},
-	system::{Handed, Input, Key, World},
-	util::Device,
-};
 use zbus::{Connection, interface, object_server::Interface, zvariant::OwnedObjectPath};
 
 pub mod input;
 pub mod play_space;
 
-enum Inputs {
-	XR {
-		controller_left: SkController,
-		controller_right: SkController,
-		hand_left: SkHand,
-		hand_right: SkHand,
-		eye_pointer: Option<EyePointer>,
-	},
-	MousePointer(MousePointer),
-	// Controllers((SkController, SkController)),
-	Hands {
-		left: SkHand,
-		right: SkHand,
-	},
-}
-
-pub struct ServerObjects {
-	connection: Connection,
-	hmd: (Arc<Spatial>, ObjectHandle<SpatialRef>),
-	play_space: Option<(Arc<Spatial>, ObjectHandle<SpatialRef>)>,
-	hand_materials: [Material; 2],
-	inputs: Inputs,
-	disable_controllers: bool,
-	disable_hands: bool,
-}
-impl ServerObjects {
-	pub fn new(
-		connection: Connection,
-		sk: &Sk,
-		hand_materials: [Material; 2],
-		disable_controllers: bool,
-		disable_hands: bool,
-	) -> ServerObjects {
-		let hmd = SpatialRef::create(&connection, "/org/stardustxr/HMD");
-
-		let play_space = Some(SpatialRef::create(&connection, "/org/stardustxr/PlaySpace"));
-		if play_space.is_some() {
-			let dbus_connection = connection.clone();
-			tokio::task::spawn(async move {
-				PlaySpaceBounds::create(&dbus_connection).await;
-				dbus_connection
-					.request_name("org.stardustxr.PlaySpace")
-					.await
-					.unwrap();
-			});
-		}
-
-		tokio::task::spawn({
-			let connection = connection.clone();
-			async move {
-				connection
-					.request_name("org.stardustxr.Controllers")
-					.await
-					.unwrap();
-				connection
-					.request_name("org.stardustxr.Hands")
-					.await
-					.unwrap();
-			}
-		});
-
-		let inputs = if sk.get_active_display_mode() == DisplayMode::MixedReality {
-			Inputs::XR {
-				controller_left: SkController::new(&connection, Handed::Left).unwrap(),
-				controller_right: SkController::new(&connection, Handed::Right).unwrap(),
-				hand_left: SkHand::new(&connection, Handed::Left).unwrap(),
-				hand_right: SkHand::new(&connection, Handed::Right).unwrap(),
-				eye_pointer: Device::has_eye_gaze()
-					.then(EyePointer::new)
-					.transpose()
-					.unwrap(),
-			}
-		} else {
-			Inputs::MousePointer(MousePointer::new().unwrap())
-		};
-
-		ServerObjects {
-			connection,
-			hmd,
-			play_space,
-			hand_materials,
-			inputs,
-			disable_controllers,
-			disable_hands,
-		}
-	}
-
-	pub fn update(
-		&mut self,
-		sk: &Sk,
-		token: &MainThreadToken,
-		dbus_connection: &Connection,
-		object_registry: &ObjectRegistry,
-	) {
-		let hmd_pose = Input::get_head();
-		self.hmd
-			.0
-			.set_local_transform(Mat4::from_scale_rotation_translation(
-				vec3(1.0, 1.0, 1.0),
-				hmd_pose.orientation.into(),
-				hmd_pose.position.into(),
-			));
-
-		if let Some(play_space) = self.play_space.as_ref() {
-			let pose = World::get_bounds_pose();
-			play_space
-				.0
-				.set_local_transform(Mat4::from_rotation_translation(
-					pose.orientation.into(),
-					pose.position.into(),
-				));
-		}
-
-		#[allow(clippy::collapsible_if)]
-		if sk.get_active_display_mode() != DisplayMode::MixedReality {
-			if Input::key(Key::F6).is_just_inactive() {
-				self.inputs = Inputs::MousePointer(MousePointer::new().unwrap());
-			}
-			// if Input::key(Key::F7).is_just_inactive() {
-			// 	self.inputs = Inputs::Controllers((
-			// 		SkController::new(Handed::Left).unwrap(),
-			// 		SkController::new(Handed::Right).unwrap(),
-			// 	));
-			// }
-			// if Input::key(Key::F8).is_just_inactive() {
-			// 	self.inputs = Inputs::Hands {
-			// 		left: SkHand::new(&self.connection, Handed::Left).unwrap(),
-			// 		right: SkHand::new(&self.connection, Handed::Right).unwrap(),
-			// 	};
-			// }
-		}
-
-		match &mut self.inputs {
-			Inputs::XR {
-				controller_left,
-				controller_right,
-				hand_left,
-				hand_right,
-				eye_pointer,
-			} => {
-				if !self.disable_controllers {
-					controller_left.update(token);
-					controller_right.update(token);
-				}
-				Input::hand_visible(Handed::Left, !self.disable_hands);
-				Input::hand_visible(Handed::Right, !self.disable_hands);
-				if !self.disable_hands {
-					hand_left.update(sk, token, &mut self.hand_materials[0]);
-					hand_right.update(sk, token, &mut self.hand_materials[1]);
-				}
-				if let Some(eye_pointer) = eye_pointer {
-					eye_pointer.update();
-				}
-			}
-			Inputs::MousePointer(mouse_pointer) => {
-				mouse_pointer.update(dbus_connection, object_registry)
-			}
-			// Inputs::Controllers((left, right)) => {
-			// 	left.update(token);
-			// 	right.update(token);
-			// }
-			Inputs::Hands { left, right } => {
-				left.update(sk, token, &mut self.hand_materials[0]);
-				right.update(sk, token, &mut self.hand_materials[1]);
-			}
-		}
-	}
-}
-
 pub struct ObjectHandle<I: Interface>(Connection, OwnedObjectPath, PhantomData<I>);
 
 impl<I: Interface> Clone for ObjectHandle<I> {

src/objects/play_space.rs

@@ -1,12 +1,150 @@
-use stereokit_rust::system::World;
+use std::sync::Arc;
+
+use bevy::prelude::*;
+use bevy_mod_openxr::{
+	helper_traits::{ToQuat, ToVec3},
+	resources::OxrFrameState,
+	session::OxrSession,
+};
+use bevy_mod_xr::{
+	session::{XrPreDestroySession, XrSessionCreated},
+	spaces::{XrPrimaryReferenceSpace, XrReferenceSpace, XrSpace},
+};
+use openxr::SpaceLocationFlags;
+use parking_lot::RwLock;
 use zbus::{Connection, ObjectServer, interface};
 
-pub struct PlaySpaceBounds;
+use crate::{DbusConnection, PreFrameWait, nodes::spatial::Spatial};
+
+use super::{ObjectHandle, SpatialRef, Tracked};
+
+pub struct PlaySpacePlugin;
+impl Plugin for PlaySpacePlugin {
+	fn build(&self, app: &mut App) {
+		app.add_systems(XrPreDestroySession, destroy_stage_space);
+		app.add_systems(XrSessionCreated, create_stage_space);
+		app.add_systems(PreFrameWait, update);
+		app.add_systems(Startup, setup);
+	}
+}
+
+fn setup(connection: Res<DbusConnection>, mut cmds: Commands) {
+	let (spatial, spatial_handle) = SpatialRef::create(&connection, "/org/stardustxr/PlaySpace");
+	// the OpenXR session might not exist quite yet
+	let tracked = Tracked::new(&connection, "/org/stardustxr/PlaySpace");
+	let dbus_connection = connection.clone();
+	let play_space_data = Arc::new(RwLock::default());
+	tokio::task::spawn({
+		let data = play_space_data.clone();
+		async move {
+			PlaySpaceBounds::create(&dbus_connection, data).await;
+			dbus_connection
+				.request_name("org.stardustxr.PlaySpace")
+				.await
+				.unwrap();
+		}
+	});
+	cmds.insert_resource(PlaySpace {
+		spatial,
+		_spatial_handle: spatial_handle,
+		tracked_handle: tracked,
+		bounds: play_space_data,
+	});
+}
+
+#[derive(Resource)]
+struct StageSpace(XrSpace);
+fn create_stage_space(session: Res<OxrSession>, mut cmds: Commands) {
+	let space = session
+		.create_reference_space(openxr::ReferenceSpaceType::STAGE, Transform::IDENTITY)
+		.inspect_err(|err| error!("failed to create Stage XrSpace"))
+		.ok();
+	if let Some(space) = space {
+		cmds.insert_resource(StageSpace(space.0));
+	}
+}
+fn destroy_stage_space(session: Res<OxrSession>, mut cmds: Commands, stage: Res<StageSpace>) {
+	session.destroy_space(stage.0);
+	cmds.remove_resource::<StageSpace>();
+}
+
+/// TODO: impl this
+fn update(
+	session: Option<Res<OxrSession>>,
+	stage: Option<Res<StageSpace>>,
+	ref_space: Option<Res<XrPrimaryReferenceSpace>>,
+	play_space: Res<PlaySpace>,
+	state: Option<Res<OxrFrameState>>,
+) {
+	let (Some(session), Some(stage), Some(ref_space), Some(state)) =
+		(session, stage, ref_space, state)
+	else {
+		play_space.bounds.write().drain(..);
+		tokio::task::spawn({
+			let handle = play_space.tracked_handle.clone();
+			async move {
+				handle.set_tracked(false);
+			}
+		});
+		return;
+	};
+	// this won't be correct with pipelined rendering
+	let location = session
+		.locate_space(&stage.0, &ref_space, state.predicted_display_time)
+		.inspect_err(|err| error!("Error while Locating OpenXR Stage Space {err}"));
+	if let Ok(location) = location {
+		let is_tracked = location.location_flags.contains(
+			SpaceLocationFlags::POSITION_VALID
+				| SpaceLocationFlags::POSITION_TRACKED
+				| SpaceLocationFlags::ORIENTATION_VALID
+				| SpaceLocationFlags::ORIENTATION_TRACKED,
+		);
+		tokio::task::spawn({
+			let handle = play_space.tracked_handle.clone();
+			async move {
+				handle.set_tracked(is_tracked);
+			}
+		});
+		if is_tracked {
+			play_space
+				.spatial
+				.set_local_transform(Mat4::from_rotation_translation(
+					location.pose.orientation.to_quat(),
+					location.pose.position.to_vec3(),
+				));
+		}
+	}
+	// session.reference_space_bounds_rect(openxr::ReferenceSpaceType::STAGE);
+
+	// if (World::has_bounds()
+	// 	&& World::get_bounds_size().x != 0.0
+	// 	&& World::get_bounds_size().y != 0.0)
+	// {
+	// 	let bounds = World::get_bounds_size();
+	// 	vec![
+	// 		((bounds.x).into(), (bounds.y).into()),
+	// 		((bounds.x).into(), (-bounds.y).into()),
+	// 		((-bounds.x).into(), (-bounds.y).into()),
+	// 		((-bounds.x).into(), (bounds.y).into()),
+	// 	]
+	// } else {
+	// 	vec![]
+	// }
+}
+
+#[derive(Resource)]
+pub struct PlaySpace {
+	spatial: Arc<Spatial>,
+	_spatial_handle: ObjectHandle<SpatialRef>,
+	tracked_handle: ObjectHandle<Tracked>,
+	bounds: Arc<RwLock<Vec<(f64, f64)>>>,
+}
+pub struct PlaySpaceBounds(Arc<RwLock<Vec<(f64, f64)>>>);
 impl PlaySpaceBounds {
-	pub async fn create(connection: &Connection) {
+	pub async fn create(connection: &Connection, data: Arc<RwLock<Vec<(f64, f64)>>>) {
 		connection
 			.object_server()
-			.at("/org/stardustxr/PlaySpace", Self)
+			.at("/org/stardustxr/PlaySpace", Self(data))
 			.await
 			.unwrap();
 	}
@@ -15,19 +153,6 @@ impl PlaySpaceBounds {
 impl PlaySpaceBounds {
 	#[zbus(property)]
 	fn bounds(&self) -> Vec<(f64, f64)> {
-		if (World::has_bounds()
-			&& World::get_bounds_size().x != 0.0
-			&& World::get_bounds_size().y != 0.0)
-		{
-			let bounds = World::get_bounds_size();
-			vec![
-				((bounds.x).into(), (bounds.y).into()),
-				((bounds.x).into(), (-bounds.y).into()),
-				((-bounds.x).into(), (-bounds.y).into()),
-				((-bounds.x).into(), (bounds.y).into()),
-			]
-		} else {
-			vec![]
-		}
+		self.0.read().clone()
 	}
 }