feat: working handtracking input methodsRUST_LOG=info,naga=warn dbus-run-session cargo run -- -e ~/build/stardust/env.sh -o 300!

Signed-off-by: Schmarni <marnistromer@gmail.com>

Cargo.lock

@@ -1179,7 +1179,7 @@ dependencies = [
 [[package]]
 name = "bevy_mod_openxr"
 version = "0.3.0"
-source = "git+https://github.com/Schmarni-Dev/bevy_openxr?branch=non_default_wait_frame_system#7a30bb2b523739501ba491956b96d9d6464ed8c5"
+source = "git+https://github.com/Schmarni-Dev/bevy_openxr?branch=non_default_wait_frame_system#f0fd9a3086f1d898cf8d1a5338a3deae7ef66e2f"
 dependencies = [
  "android_system_properties",
  "ash",
@@ -1197,7 +1197,7 @@ dependencies = [
 [[package]]
 name = "bevy_mod_xr"
 version = "0.3.0"
-source = "git+https://github.com/Schmarni-Dev/bevy_openxr?branch=non_default_wait_frame_system#7a30bb2b523739501ba491956b96d9d6464ed8c5"
+source = "git+https://github.com/Schmarni-Dev/bevy_openxr?branch=non_default_wait_frame_system#f0fd9a3086f1d898cf8d1a5338a3deae7ef66e2f"
 dependencies = [
  "bevy",
 ]
@@ -1696,7 +1696,7 @@ dependencies = [
  "bitflags 2.9.1",
  "cexpr",
  "clang-sys",
- "itertools 0.13.0",
+ "itertools 0.11.0",
  "log",
  "prettyplease",
  "proc-macro2",
@@ -3550,15 +3550,6 @@ dependencies = [
  "either",
 ]
 
-[[package]]
-name = "itertools"
-version = "0.13.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
-dependencies = [
- "either",
-]
-
 [[package]]
 name = "itertools"
 version = "0.14.0"
@@ -4259,7 +4250,7 @@ version = "0.7.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "af1844ef2428cc3e1cb900be36181049ef3d3193c63e43026cfe202983b27a56"
 dependencies = [
- "proc-macro-crate 3.3.0",
+ "proc-macro-crate 1.3.1",
  "proc-macro2",
  "quote",
  "syn 2.0.101",
@@ -4971,7 +4962,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d"
 dependencies = [
  "anyhow",
- "itertools 0.14.0",
+ "itertools 0.11.0",
  "proc-macro2",
  "quote",
  "syn 2.0.101",
@@ -6925,7 +6916,7 @@ version = "0.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb"
 dependencies = [
- "windows-sys 0.59.0",
+ "windows-sys 0.48.0",
 ]
 
 [[package]]

src/main.rs

@@ -12,10 +12,11 @@ use crate::nodes::{audio, drawable, input};
 
 use bevy::MinimalPlugins;
 use bevy::a11y::AccessibilityPlugin;
-use bevy::app::{App, ScheduleRunnerPlugin, TerminalCtrlCHandlerPlugin};
+use bevy::app::{App, MainScheduleOrder, ScheduleRunnerPlugin, TerminalCtrlCHandlerPlugin};
 use bevy::asset::{AssetMetaCheck, UnapprovedPathMode};
 use bevy::audio::AudioPlugin;
 use bevy::core_pipeline::CorePipelinePlugin;
+use bevy::ecs::schedule::{ExecutorKind, ScheduleLabel};
 use bevy::gizmos::GizmoPlugin;
 use bevy::gltf::GltfPlugin;
 use bevy::input::{InputPlugin, InputSystem};
@@ -27,14 +28,21 @@ use bevy::scene::ScenePlugin;
 use bevy::text::FontLoader;
 use bevy::winit::{WakeUp, WinitPlugin};
 use bevy_mod_meshtext::MeshTextPlugin;
+use bevy_mod_openxr::action_binding::OxrActionBindingPlugin;
+use bevy_mod_openxr::action_set_attaching::OxrActionAttachingPlugin;
+use bevy_mod_openxr::action_set_syncing::OxrActionSyncingPlugin;
 use bevy_mod_openxr::add_xr_plugins;
 use bevy_mod_openxr::exts::OxrExtensions;
+use bevy_mod_openxr::features::handtracking::HandTrackingPlugin;
 use bevy_mod_openxr::features::overlay::OxrOverlaySettings;
-use bevy_mod_openxr::init::OxrInitPlugin;
+use bevy_mod_openxr::features::passthrough::OxrPassthroughPlugin;
+use bevy_mod_openxr::init::{OxrInitPlugin, should_run_frame_loop};
 use bevy_mod_openxr::reference_space::OxrReferenceSpacePlugin;
-use bevy_mod_openxr::resources::OxrSessionConfig;
+use bevy_mod_openxr::render::{OxrRenderPlugin, OxrWaitFrameSystem};
+use bevy_mod_openxr::resources::{OxrFrameState, OxrFrameWaiter, OxrSessionConfig};
 use bevy_mod_openxr::types::AppInfo;
 use bevy_mod_xr::hand_debug_gizmos::HandGizmosPlugin;
+use bevy_mod_xr::session::{XrFirst, XrHandleEvents, session_running};
 use clap::Parser;
 use core::client::{Client, tick_internal_client};
 use core::task;
@@ -42,6 +50,8 @@ use directories::ProjectDirs;
 use nodes::drawable::model::ModelNodePlugin;
 use nodes::spatial::SpatialNodePlugin;
 use objects::ServerObjects;
+use objects::input::sk_hand::HandPlugin;
+use objects::play_space::PlaySpacePlugin;
 use openxr::{EnvironmentBlendMode, ReferenceSpaceType};
 use session::{launch_start, save_session};
 use stardust_xr::schemas::dbus::object_registry::ObjectRegistry;
@@ -239,6 +249,13 @@ async fn main() {
 static DEFAULT_SKYTEX: OnceLock<Tex> = OnceLock::new();
 static DEFAULT_SKYLIGHT: OnceLock<SphericalHarmonics> = OnceLock::new();
 
+#[derive(ScheduleLabel, Hash, Debug, PartialEq, Eq, Clone, Copy)]
+pub struct PreFrameWait;
+#[derive(Resource, Deref)]
+pub struct ObjectRegistryRes(ObjectRegistry);
+#[derive(Resource, Deref)]
+pub struct DbusConnection(Connection);
+
 fn bevy_loop(
 	ready_notifier: Arc<Notify>,
 	project_dirs: Option<ProjectDirs>,
@@ -247,6 +264,7 @@ fn bevy_loop(
 	object_registry: ObjectRegistry,
 ) {
 	let mut app = App::new();
+	app.insert_resource(DbusConnection(dbus_connection));
 	app.add_plugins(AssetPlugin {
 		meta_check: AssetMetaCheck::Never,
 		unapproved_path_mode: UnapprovedPathMode::Allow,
@@ -254,18 +272,18 @@ fn bevy_loop(
 	});
 	let mut plugins = MinimalPlugins
 		.build()
-		.disable::<ScheduleRunnerPlugin>()
+		// .disable::<ScheduleRunnerPlugin>()
 		.add(TransformPlugin)
 		.add(InputPlugin)
 		/* .add(AccessibilityPlugin) */;
 	// TODO: figure out headless
-	{
+	// {
-		plugins = plugins.add(WindowPlugin::default()).add({
+	// 	plugins = plugins.add(WindowPlugin::default()).add({
-			let mut winit = WinitPlugin::<WakeUp>::default();
+	// 		let mut winit = WinitPlugin::<WakeUp>::default();
-			winit.run_on_any_thread = true;
+	// 		winit.run_on_any_thread = true;
-			winit
+	// 		winit
-		});
+	// 	});
-	}
+	// }
 	plugins = plugins
 		.add(TerminalCtrlCHandlerPlugin)
 		// bevy_mod_openxr will replace this, TODO: figure out how to mix this with
@@ -288,8 +306,23 @@ fn bevy_loop(
 		.add(AudioPlugin::default())
 		.add(GizmoPlugin)
 		.add(AccessibilityPlugin);
+	let mut task_pool_plugin = TaskPoolPlugin::default();
+	// make tokio work
+	let handle = tokio::runtime::Handle::current();
+	let enter_runtime_context = Arc::new(move || {
+		// TODO: this might be a memory leak
+		std::mem::forget(handle.enter());
+	});
+	task_pool_plugin.task_pool_options.io.on_thread_spawn = Some(enter_runtime_context.clone());
+	task_pool_plugin.task_pool_options.compute.on_thread_spawn =
+		Some(enter_runtime_context.clone());
+	task_pool_plugin
+		.task_pool_options
+		.async_compute
+		.on_thread_spawn = Some(enter_runtime_context.clone());
+	plugins = plugins.set(task_pool_plugin);
 	app.add_plugins(
-		add_xr_plugins(plugins)
+		add_xr_plugins(plugins.add(WindowPlugin::default()))
 			.set(OxrInitPlugin {
 				app_info: AppInfo {
 					name: "Stardust XR".into(),
@@ -306,17 +339,21 @@ fn bevy_loop(
 				},
 				..default()
 			})
+			.set(OxrRenderPlugin {
+				default_wait_frame: false,
+				..default()
+			})
 			.set(OxrReferenceSpacePlugin {
 				default_primary_ref_space: ReferenceSpaceType::LOCAL,
-			}),
+			})
+			// Disable a bunch of unneeded plugins
+			// this plugin uses the fb extention, blend mode still works
+			.disable::<OxrPassthroughPlugin>()
+			// we don't do any action stuff that needs to integrate with the ecosystem
+			.disable::<OxrActionAttachingPlugin>()
+			.disable::<OxrActionSyncingPlugin>()
+			.disable::<OxrActionBindingPlugin>(),
 	);
-	app.init_asset::<Font>().init_asset_loader::<FontLoader>();
-	if let Some(priority) = args.overlay_priority {
-		app.insert_resource(OxrOverlaySettings {
-			session_layer_placement: priority,
-			..default()
-		});
-	}
 	app.add_plugins((
 		bevy_sk::hand::HandPlugin,
 		bevy_sk::vr_materials::SkMaterialPlugin {
@@ -325,7 +362,14 @@ fn bevy_loop(
 		bevy_sk::skytext::SphericalHarmonicsPlugin,
 	));
 	app.add_plugins(HandGizmosPlugin);
-	app.add_plugins(MeshTextPlugin);
+	// app.add_plugins(MeshTextPlugin);
+	// app.init_asset::<Font>().init_asset_loader::<FontLoader>();
+	if let Some(priority) = args.overlay_priority {
+		app.insert_resource(OxrOverlaySettings {
+			session_layer_placement: priority,
+			..default()
+		});
+	}
 	app.insert_resource(OxrSessionConfig {
 		blend_modes: Some(vec![
 			EnvironmentBlendMode::ALPHA_BLEND,
@@ -334,26 +378,63 @@ fn bevy_loop(
 		]),
 		..default()
 	});
+	let mut pre_frame_wait = Schedule::new(PreFrameWait);
+	pre_frame_wait.set_executor_kind(ExecutorKind::MultiThreaded);
+	app.add_schedule(pre_frame_wait);
 	app.insert_resource(ClearColor(Color::BLACK.with_alpha(0.0)));
+	app.insert_resource(ObjectRegistryRes(object_registry));
 	app.add_plugins((RemotePlugin::default(), RemoteHttpPlugin::default()));
-	app.add_plugins((SpatialNodePlugin, ModelNodePlugin));
+	// the Stardust server plugins
+	app.add_plugins((
+		SpatialNodePlugin,
+		ModelNodePlugin,
+		PlaySpacePlugin,
+		HandPlugin,
+	));
 	ready_notifier.notify_waiters();
-	app.add_systems(PreUpdate, main_loop_system.after(InputSystem));
+	app.add_systems(
+		XrFirst,
+		xr_step
+			.in_set(OxrWaitFrameSystem)
+			.in_set(XrHandleEvents::FrameLoop),
+	);
 	app.run();
 }
 
-fn main_loop_system(world: &mut World) {
+fn xr_step(world: &mut World) {
 	// camera::update(token);
 	#[cfg(feature = "wayland")]
 	wayland.frame_event();
 	destroy_queue::clear();
 
-	// objects.update(&sk, token, &dbus_connection, &object_registry);
+	// update things like the Xr input methods
-	// input::process_input();
+	world.run_schedule(PreFrameWait);
+	input::process_input();
 	let time = world.resource::<bevy::prelude::Time>().delta_secs_f64();
 	nodes::root::Root::send_frame_events(time);
 
-	// Wait
+	// we are targeting the frame after the wait
+	if let Some(mut state) = world.get_resource_mut::<OxrFrameState>() {
+		state.predicted_display_time = openxr::Time::from_nanos(
+			state.predicted_display_time.as_nanos() + state.predicted_display_period.as_nanos(),
+		);
+	}
+
+	let should_wait = world
+		.run_system_cached(should_run_frame_loop)
+		.unwrap_or(false);
+	if should_wait {
+		world.resource_scope::<OxrFrameWaiter, _>(|world, mut waiter| {
+			let state = waiter
+				.wait()
+				.inspect_err(|err| error!("failed to wait OpenXR frame: {err}"))
+				.ok();
+
+			if let Some(state) = state {
+				world.insert_resource(OxrFrameState(state));
+			}
+		});
+	}
 
 	tick_internal_client();
 	#[cfg(feature = "wayland")]

src/objects/input/sk_hand.rs

@@ -1,15 +1,27 @@
 use crate::core::client::INTERNAL_CLIENT;
 use crate::nodes::OwnedNode;
 use crate::nodes::fields::{Field, FieldTrait};
-use crate::nodes::input::{INPUT_HANDLER_REGISTRY, InputDataType, InputHandler};
+use crate::nodes::input::{Finger, INPUT_HANDLER_REGISTRY, InputDataType, InputHandler, Thumb};
 use crate::nodes::{
 	Node,
 	input::{Hand, InputMethod, Joint},
 	spatial::Spatial,
 };
 use crate::objects::{ObjectHandle, SpatialRef, Tracked};
+use crate::{DbusConnection, ObjectRegistryRes, PreFrameWait};
+use bevy::prelude::Transform as BevyTransform;
+use bevy::prelude::*;
+use bevy_mod_openxr::helper_traits::{ToQuat, ToVec3};
+use bevy_mod_openxr::resources::OxrFrameState;
+use bevy_mod_openxr::session::OxrSession;
+use bevy_mod_xr::hands::{HandBone, HandSide, XrHandBoneEntities, XrHandBoneRadius};
+use bevy_mod_xr::session::{XrPreDestroySession, XrSessionCreated};
+use bevy_mod_xr::spaces::{XrPrimaryReferenceSpace, XrSpaceLocationFlags};
+use bevy_sk::hand::GRADIENT_TEXTURE_HANDLE;
+use bevy_sk::vr_materials::PbrMaterial;
 use color_eyre::eyre::Result;
 use glam::{Mat4, Quat, Vec3};
+use openxr::{HandJointLocation, SpaceLocationFlags};
 use serde::{Deserialize, Serialize};
 use stardust_xr::values::Datamap;
 use std::sync::Arc;
@@ -21,15 +33,146 @@ use zbus::Connection;
 
 use super::{CaptureManager, get_sorted_handlers};
 
-fn convert_joint(joint: HandJoint) -> Joint {
+pub struct HandPlugin;
+impl Plugin for HandPlugin {
+	fn build(&self, app: &mut App) {
+		app.add_systems(PreFrameWait, update_hands);
+		app.add_systems(XrSessionCreated, create_trackers);
+		app.add_systems(XrPreDestroySession, destroy_trackers);
+		app.add_systems(PostUpdate, update_hand_material);
+		app.add_systems(Startup, setup);
+	}
+}
+fn update_hands(
+	mut hands: ResMut<Hands>,
+	session: Option<Res<OxrSession>>,
+	state: Option<Res<OxrFrameState>>,
+	ref_space: Option<Res<XrPrimaryReferenceSpace>>,
+	mut materials: ResMut<Assets<PbrMaterial>>,
+	mut joint_query: Query<(
+		&mut BevyTransform,
+		&mut XrSpaceLocationFlags,
+		&mut XrHandBoneRadius,
+	)>,
+	joints_query: Query<&XrHandBoneEntities>,
+) {
+	let (Some(session), Some(state), Some(ref_space)) = (session, state, ref_space) else {
+		tokio::task::spawn({
+			let left = hands.left.tracked.clone();
+			let right = hands.right.tracked.clone();
+			async move {
+				left.set_tracked(false);
+				right.set_tracked(false);
+			}
+		});
+		return;
+	};
+	let get_joints = |hand: &mut SkHand| -> Option<openxr::HandJointLocations> {
+		let Some(tracker) = hand.tracker.as_ref() else {
+			hand.input.spatial.node().unwrap().set_enabled(false);
+			let handle = hand.tracked.clone();
+			tokio::task::spawn(async move {
+				handle.set_tracked(false);
+			});
+			return None;
+		};
+		// this won't be correct with pipelined rendering
+		session
+			.locate_hand_joints(tracker, &ref_space, state.predicted_display_time)
+			.inspect_err(|err| error!("Error while locating hand joints"))
+			.ok()
+			.flatten()
+	};
+	let joints_left = get_joints(&mut hands.left);
+	let joints_right = get_joints(&mut hands.right);
+	hands.left.update(joints_left.as_ref(), &mut materials);
+	hands.right.update(joints_right.as_ref(), &mut materials);
+}
+
+fn pinch_between(joint_1: &Joint, joint_2: &Joint) -> f32 {
+	const PINCH_MAX: f32 = 0.11;
+	const PINCH_ACTIVACTION_DISTANCE: f32 = 0.01;
+	let combined_radius = joint_1.radius + joint_2.radius;
+	let pinch_dist =
+		Vec3::from(joint_1.position).distance(Vec3::from(joint_2.position)) - combined_radius;
+	(1.0 - ((pinch_dist - PINCH_ACTIVACTION_DISTANCE) / (PINCH_MAX - PINCH_ACTIVACTION_DISTANCE)))
+		.clamp(0.0, 1.0)
+}
+
+fn create_trackers(session: Res<OxrSession>, mut hands: ResMut<Hands>) {
+	hands.left.tracker = session
+		.create_hand_tracker(openxr::HandEXT::LEFT)
+		.inspect_err(|err| error!("failed to create left hand tracker"))
+		.ok();
+	hands.right.tracker = session
+		.create_hand_tracker(openxr::HandEXT::RIGHT)
+		.inspect_err(|err| error!("failed to create right hand tracker"))
+		.ok();
+}
+fn destroy_trackers(mut hands: ResMut<Hands>) {
+	hands.left.tracker.take();
+	hands.right.tracker.take();
+}
+#[derive(Component)]
+struct CorrectHandMaterial;
+fn update_hand_material(
+	query: Query<
+		(Entity, &HandSide),
+		(
+			With<XrHandBoneEntities>,
+			With<MeshMaterial3d<PbrMaterial>>,
+			Without<CorrectHandMaterial>,
+		),
+	>,
+	mut cmds: Commands,
+	hands: Res<Hands>,
+) {
+	for (entity, side) in &query {
+		let handle = match side {
+			HandSide::Left => hands.left.material.clone(),
+			HandSide::Right => hands.right.material.clone(),
+		};
+		cmds.entity(entity)
+			.insert(MeshMaterial3d(handle))
+			.insert(CorrectHandMaterial);
+	}
+}
+
+fn setup(
+	connection: Res<DbusConnection>,
+	mut cmds: Commands,
+	mut materials: ResMut<Assets<PbrMaterial>>,
+) {
+	tokio::task::spawn({
+		let connection = connection.clone();
+		async move {
+			connection
+				.request_name("org.stardustxr.Hands")
+				.await
+				.unwrap();
+		}
+	});
+	cmds.insert_resource(Hands {
+		left: SkHand::new(&connection, HandSide::Left, &mut materials).unwrap(),
+		right: SkHand::new(&connection, HandSide::Right, &mut materials).unwrap(),
+	});
+}
+
+fn convert_joint(joint: HandJointLocation) -> Joint {
 	Joint {
-		position: Vec3::from(joint.position).into(),
+		position: joint.pose.position.to_vec3().into(),
-		rotation: Quat::from(joint.orientation).into(),
+		rotation: joint.pose.orientation.to_quat().into(),
 		radius: joint.radius,
 		distance: 0.0,
 	}
 }
 
+#[derive(Resource)]
+struct Hands {
+	left: SkHand,
+	right: SkHand,
+}
+
 #[derive(Default, Deserialize, Serialize)]
 struct HandDatamap {
 	pinch_strength: f32,
@@ -40,117 +183,158 @@ pub struct SkHand {
 	_node: OwnedNode,
 	palm_spatial: Arc<Spatial>,
 	palm_object: ObjectHandle<SpatialRef>,
-	handed: Handed,
+	side: HandSide,
 	input: Arc<InputMethod>,
 	capture_manager: CaptureManager,
 	datamap: HandDatamap,
 	tracked: ObjectHandle<Tracked>,
+	tracker: Option<openxr::HandTracker>,
+	captured: bool,
+	material: Handle<PbrMaterial>,
 }
 impl SkHand {
-	pub fn new(connection: &Connection, handed: Handed) -> Result<Self> {
+	pub fn new(
+		connection: &Connection,
+		side: HandSide,
+		materials: &mut Assets<PbrMaterial>,
+	) -> Result<Self> {
 		let (palm_spatial, palm_object) = SpatialRef::create(
 			connection,
 			&("/org/stardustxr/Hand/".to_string()
-				+ match handed {
+				+ match side {
-					Handed::Left => "left",
+					HandSide::Left => "left",
-					_ => "right",
+					HandSide::Right => "right",
 				} + "/palm"),
 		);
 		let tracked = Tracked::new(
 			connection,
 			&("/org/stardustxr/Hand/".to_string()
-				+ match handed {
+				+ match side {
-					Handed::Left => "left",
+					HandSide::Left => "left",
-					_ => "right",
+					HandSide::Right => "right",
 				}),
 		);
-		let _node = Node::generate(&INTERNAL_CLIENT, false).add_to_scenegraph_owned()?;
+		let node = Node::generate(&INTERNAL_CLIENT, false).add_to_scenegraph_owned()?;
-		Spatial::add_to(&_node.0, None, Mat4::IDENTITY, false);
+		Spatial::add_to(&node.0, None, Mat4::IDENTITY, false);
 		let hand = InputDataType::Hand(Hand {
-			right: handed == Handed::Right,
+			right: matches!(side, HandSide::Right),
 			..Default::default()
 		});
 		let datamap = Datamap::from_typed(HandDatamap::default())?;
-		let input = InputMethod::add_to(&_node.0, hand, datamap)?;
+		let input = InputMethod::add_to(&node.0, hand, datamap)?;
-		Input::hand_visible(handed, true);
 
+		let material = materials.add(PbrMaterial {
+			color: Srgba::new(1.0, 1.0, 1.0, 1.0).into(),
+			alpha_mode: AlphaMode::Blend,
+			use_stereokit_uvs: false,
+			diffuse_texture: Some(GRADIENT_TEXTURE_HANDLE),
+			roughness: 1.0,
+			..default()
+		});
 		Ok(SkHand {
-			_node,
+			_node: node,
 			palm_spatial,
 			palm_object,
-			handed,
+			side,
 			input,
 			tracked,
 			capture_manager: CaptureManager::default(),
 			datamap: Default::default(),
+			tracker: None,
+			material,
+			captured: false,
 		})
 	}
-	pub fn update(&mut self, sk: &Sk, token: &MainThreadToken, material: &mut Material) {
+	fn update(
-		let sk_hand = Input::hand(self.handed);
+		&mut self,
-		let real_hand = Input::hand_source(self.handed) as u32 == HandSource::Articulated as u32;
+		joints: Option<&openxr::HandJointLocations>,
-		if let InputDataType::Hand(hand) = &mut *self.input.data.lock() {
+		materials: &mut ResMut<Assets<PbrMaterial>>,
-			let input_node = self.input.spatial.node().unwrap();
+	) {
-			input_node.set_enabled(
+		// TODO: use the hand data source ext
-				(real_hand || sk.get_active_display_mode() == DisplayMode::Flatscreen)
+		let real_hand = true;
-					&& sk_hand.tracked.is_active(),
+		let input_node = self.input.spatial.node().unwrap();
-			);
+		let is_tracked = real_hand
-			let enabled = input_node.enabled();
+			&& joints.is_some_and(|v| {
-			tokio::spawn({
+				v.iter().all(|v| {
-				// this is suboptimal since it probably allocates a fresh string every frame
+					v.location_flags.contains(
-				let handle = self.tracked.clone();
+						SpaceLocationFlags::POSITION_VALID
-				async move {
+							| SpaceLocationFlags::POSITION_TRACKED
-					handle.set_tracked(enabled).await;
+							| SpaceLocationFlags::ORIENTATION_VALID
-				}
+							| SpaceLocationFlags::ORIENTATION_TRACKED,
+					)
+				})
 			});
-			if enabled {
+		input_node.set_enabled(is_tracked);
-				hand.thumb.tip = convert_joint(sk_hand.fingers[0][4]);
+		tokio::task::spawn({
-				hand.thumb.distal = convert_joint(sk_hand.fingers[0][3]);
+			let handle = self.tracked.clone();
-				hand.thumb.proximal = convert_joint(sk_hand.fingers[0][2]);
+			async move {
-				hand.thumb.metacarpal = convert_joint(sk_hand.fingers[0][1]);
+				handle.set_tracked(is_tracked);
-
-				for (finger, mut sk_finger) in [
-					(&mut hand.index, sk_hand.fingers[1]),
-					(&mut hand.middle, sk_hand.fingers[2]),
-					(&mut hand.ring, sk_hand.fingers[3]),
-					(&mut hand.little, sk_hand.fingers[4]),
-				] {
-					sk_finger[4].radius = 0.0;
-					finger.tip = convert_joint(sk_finger[4]);
-					finger.distal = convert_joint(sk_finger[3]);
-					finger.intermediate = convert_joint(sk_finger[2]);
-					finger.proximal = convert_joint(sk_finger[1]);
-					finger.metacarpal = convert_joint(sk_finger[0]);
-				}
-
-				hand.palm.position = Vec3::from(sk_hand.palm.position).into();
-				hand.palm.rotation = Quat::from(sk_hand.palm.orientation).into();
-				hand.palm.radius =
-					(sk_hand.fingers[2][0].radius + sk_hand.fingers[2][1].radius) * 0.5;
-
-				self.palm_spatial
-					.set_local_transform(Mat4::from_rotation_translation(
-						hand.palm.rotation.into(),
-						hand.palm.position.into(),
-					));
-
-				hand.wrist.position = Vec3::from(sk_hand.wrist.position).into();
-				hand.wrist.rotation = Quat::from(sk_hand.wrist.orientation).into();
-				hand.wrist.radius =
-					(sk_hand.fingers[0][0].radius + sk_hand.fingers[4][0].radius) * 0.5;
-
-				hand.elbow = None;
-
-				let hand_color = if self.capture_manager.capture.upgrade().is_none() {
-					Color128::new_rgb(1.0, 1.0, 1.0)
-				} else {
-					Color128::new_rgb(0.0, 1.0, 0.75)
-				};
-				material.color_tint(hand_color);
 			}
+		});
+		if is_tracked {
+			// cannot ever crash, is_tracked is only true of joints is some
+			let joints = joints.unwrap();
+			let new_hand = Hand {
+				right: matches!(self.side, HandSide::Right),
+				thumb: Thumb {
+					tip: convert_joint(joints[HandBone::ThumbTip as usize]),
+					distal: convert_joint(joints[HandBone::ThumbDistal as usize]),
+					proximal: convert_joint(joints[HandBone::ThumbProximal as usize]),
+					metacarpal: convert_joint(joints[HandBone::ThumbMetacarpal as usize]),
+				},
+				index: Finger {
+					tip: convert_joint(joints[HandBone::IndexTip as usize]),
+					distal: convert_joint(joints[HandBone::IndexDistal as usize]),
+					intermediate: convert_joint(joints[HandBone::IndexIntermediate as usize]),
+					proximal: convert_joint(joints[HandBone::IndexProximal as usize]),
+					metacarpal: convert_joint(joints[HandBone::IndexMetacarpal as usize]),
+				},
+				middle: Finger {
+					tip: convert_joint(joints[HandBone::MiddleTip as usize]),
+					distal: convert_joint(joints[HandBone::MiddleDistal as usize]),
+					intermediate: convert_joint(joints[HandBone::MiddleIntermediate as usize]),
+					proximal: convert_joint(joints[HandBone::MiddleProximal as usize]),
+					metacarpal: convert_joint(joints[HandBone::MiddleMetacarpal as usize]),
+				},
+				ring: Finger {
+					tip: convert_joint(joints[HandBone::RingTip as usize]),
+					distal: convert_joint(joints[HandBone::RingDistal as usize]),
+					intermediate: convert_joint(joints[HandBone::RingIntermediate as usize]),
+					proximal: convert_joint(joints[HandBone::RingProximal as usize]),
+					metacarpal: convert_joint(joints[HandBone::RingMetacarpal as usize]),
+				},
+				little: Finger {
+					tip: convert_joint(joints[HandBone::LittleTip as usize]),
+					distal: convert_joint(joints[HandBone::LittleDistal as usize]),
+					intermediate: convert_joint(joints[HandBone::LittleIntermediate as usize]),
+					proximal: convert_joint(joints[HandBone::LittleProximal as usize]),
+					metacarpal: convert_joint(joints[HandBone::LittleMetacarpal as usize]),
+				},
+				palm: convert_joint(joints[HandBone::Palm as usize]),
+				wrist: convert_joint(joints[HandBone::Wrist as usize]),
+				elbow: None,
+			};
+			self.palm_spatial
+				.set_local_transform(Mat4::from_rotation_translation(
+					new_hand.palm.rotation.into(),
+					new_hand.palm.position.into(),
+				));
+
+			self.datamap.pinch_strength = pinch_between(&new_hand.thumb.tip, &new_hand.index.tip);
+			// this is how stereokit calculates grab
+			self.datamap.grab_strength =
+				pinch_between(&new_hand.ring.tip, &new_hand.ring.metacarpal);
+
+			*self.input.data.lock() = InputDataType::Hand(new_hand);
+			*self.input.datamap.lock() = Datamap::from_typed(&self.datamap).unwrap();
+			let captured = self.capture_manager.capture.upgrade().is_some();
+			if captured && !self.captured {
+				materials.get_mut(&self.material).unwrap().color = Srgba::rgb(0., 1., 0.75).into();
+			} else if self.captured && !captured {
+				materials.get_mut(&self.material).unwrap().color = Srgba::rgb(1., 1.0, 1.0).into();
+			}
+			self.captured = captured;
 		}
-		self.datamap.pinch_strength = sk_hand.pinch_activation;
-		self.datamap.grab_strength = sk_hand.grip_activation;
-		*self.input.datamap.lock() = Datamap::from_typed(&self.datamap).unwrap();
 
 		let distance_calculator = |space: &Arc<Spatial>, data: &InputDataType, field: &Field| {
 			let InputDataType::Hand(hand) = data else {
@@ -183,11 +367,6 @@ impl SkHand {
 			.set_handler_order(sorted_handlers.iter().map(|(handler, _)| handler));
 	}
 }
-impl Drop for SkHand {
-	fn drop(&mut self) {
-		Input::hand_visible(self.handed, false);
-	}
-}
 
 fn joint_to_line_point(joint: &Joint, color: Color128) -> LinePoint {
 	LinePoint {

src/objects/mod.rs

@@ -13,6 +13,7 @@ use input::{
 	eye_pointer::EyePointer, mouse_pointer::MousePointer, sk_controller::SkController,
 	sk_hand::SkHand,
 };
+use parking_lot::RwLock;
 use play_space::PlaySpaceBounds;
 use stardust_xr::schemas::dbus::object_registry::ObjectRegistry;
 use std::{
@@ -34,16 +35,16 @@ enum Inputs {
 	XR {
 		controller_left: SkController,
 		controller_right: SkController,
-		hand_left: SkHand,
+		// hand_left: SkHand,
-		hand_right: SkHand,
+		// hand_right: SkHand,
 		eye_pointer: Option<EyePointer>,
 	},
 	MousePointer(MousePointer),
 	// Controllers((SkController, SkController)),
-	Hands {
+	// Hands {
-		left: SkHand,
+	// 	left: SkHand,
-		right: SkHand,
+	// 	right: SkHand,
-	},
+	// },
 }
 
 pub struct ServerObjects {
@@ -69,7 +70,8 @@ impl ServerObjects {
 		if play_space.is_some() {
 			let dbus_connection = connection.clone();
 			tokio::task::spawn(async move {
-				PlaySpaceBounds::create(&dbus_connection).await;
+				let play_space_data = Arc::new(RwLock::default());
+				PlaySpaceBounds::create(&dbus_connection, play_space_data).await;
 				dbus_connection
 					.request_name("org.stardustxr.PlaySpace")
 					.await
@@ -95,8 +97,8 @@ impl ServerObjects {
 			Inputs::XR {
 				controller_left: SkController::new(&connection, Handed::Left).unwrap(),
 				controller_right: SkController::new(&connection, Handed::Right).unwrap(),
-				hand_left: SkHand::new(&connection, Handed::Left).unwrap(),
+				// hand_left: SkHand::new(&connection, Handed::Left).unwrap(),
-				hand_right: SkHand::new(&connection, Handed::Right).unwrap(),
+				// hand_right: SkHand::new(&connection, Handed::Right).unwrap(),
 				eye_pointer: Device::has_eye_gaze()
 					.then(EyePointer::new)
 					.transpose()
@@ -166,8 +168,6 @@ impl ServerObjects {
 			Inputs::XR {
 				controller_left,
 				controller_right,
-				hand_left,
-				hand_right,
 				eye_pointer,
 			} => {
 				if !self.disable_controllers {
@@ -176,10 +176,10 @@ impl ServerObjects {
 				}
 				Input::hand_visible(Handed::Left, !self.disable_hands);
 				Input::hand_visible(Handed::Right, !self.disable_hands);
-				if !self.disable_hands {
+				// if !self.disable_hands {
-					hand_left.update(sk, token, &mut self.hand_materials[0]);
+				// 	hand_left.update(sk, token, &mut self.hand_materials[0]);
-					hand_right.update(sk, token, &mut self.hand_materials[1]);
+				// 	hand_right.update(sk, token, &mut self.hand_materials[1]);
-				}
+				// }
 				if let Some(eye_pointer) = eye_pointer {
 					eye_pointer.update();
 				}
@@ -191,10 +191,10 @@ impl ServerObjects {
 			// 	left.update(token);
 			// 	right.update(token);
 			// }
-			Inputs::Hands { left, right } => {
+			// Inputs::Hands { left, right } => {
-				left.update(sk, token, &mut self.hand_materials[0]);
+			// 	left.update(sk, token, &mut self.hand_materials[0]);
-				right.update(sk, token, &mut self.hand_materials[1]);
+			// 	right.update(sk, token, &mut self.hand_materials[1]);
-			}
+			// }
 		}
 	}
 }

src/objects/play_space.rs

@@ -1,12 +1,151 @@
+use std::sync::Arc;
+
+use bevy::prelude::*;
+use bevy_mod_openxr::{
+	helper_traits::{ToQuat, ToVec3},
+	resources::OxrFrameState,
+	session::OxrSession,
+};
+use bevy_mod_xr::{
+	session::{XrPreDestroySession, XrSessionCreated},
+	spaces::{XrPrimaryReferenceSpace, XrReferenceSpace, XrSpace},
+};
+use openxr::SpaceLocationFlags;
+use parking_lot::RwLock;
 use stereokit_rust::system::World;
 use zbus::{Connection, ObjectServer, interface};
 
-pub struct PlaySpaceBounds;
+use crate::{DbusConnection, PreFrameWait, nodes::spatial::Spatial};
+
+use super::{ObjectHandle, SpatialRef, Tracked};
+
+pub struct PlaySpacePlugin;
+impl Plugin for PlaySpacePlugin {
+	fn build(&self, app: &mut App) {
+		app.add_systems(XrPreDestroySession, destroy_stage_space);
+		app.add_systems(XrSessionCreated, create_stage_space);
+		app.add_systems(PreFrameWait, update);
+		app.add_systems(Startup, setup);
+	}
+}
+
+fn setup(connection: Res<DbusConnection>, mut cmds: Commands) {
+	let (spatial, spatial_handle) = SpatialRef::create(&connection, "/org/stardustxr/PlaySpace");
+	// the OpenXR session might not exist quite yet
+	let tracked = Tracked::new(&connection, "/org/stardustxr/PlaySpace");
+	let dbus_connection = connection.clone();
+	let play_space_data = Arc::new(RwLock::default());
+	tokio::task::spawn({
+		let data = play_space_data.clone();
+		async move {
+			PlaySpaceBounds::create(&dbus_connection, data).await;
+			dbus_connection
+				.request_name("org.stardustxr.PlaySpace")
+				.await
+				.unwrap();
+		}
+	});
+	cmds.insert_resource(PlaySpace {
+		spatial,
+		_spatial_handle: spatial_handle,
+		tracked_handle: tracked,
+		bounds: play_space_data,
+	});
+}
+
+#[derive(Resource)]
+struct StageSpace(XrSpace);
+fn create_stage_space(session: Res<OxrSession>, mut cmds: Commands) {
+	let space = session
+		.create_reference_space(openxr::ReferenceSpaceType::STAGE, Transform::IDENTITY)
+		.inspect_err(|err| error!("failed to create Stage XrSpace"))
+		.ok();
+	if let Some(space) = space {
+		cmds.insert_resource(StageSpace(space.0));
+	}
+}
+fn destroy_stage_space(session: Res<OxrSession>, mut cmds: Commands, stage: Res<StageSpace>) {
+	session.destroy_space(stage.0);
+	cmds.remove_resource::<StageSpace>();
+}
+
+/// TODO: impl this
+fn update(
+	session: Option<Res<OxrSession>>,
+	stage: Option<Res<StageSpace>>,
+	ref_space: Option<Res<XrPrimaryReferenceSpace>>,
+	play_space: Res<PlaySpace>,
+	state: Option<Res<OxrFrameState>>,
+) {
+	let (Some(session), Some(stage), Some(ref_space), Some(state)) =
+		(session, stage, ref_space, state)
+	else {
+		play_space.bounds.write().drain(..);
+		tokio::task::spawn({
+			let handle = play_space.tracked_handle.clone();
+			async move {
+				handle.set_tracked(false);
+			}
+		});
+		return;
+	};
+	// this won't be correct with pipelined rendering
+	let location = session
+		.locate_space(&stage.0, &ref_space, state.predicted_display_time)
+		.inspect_err(|err| error!("Error while Locating OpenXR Stage Space {err}"));
+	if let Ok(location) = location {
+		let is_tracked = location.location_flags.contains(
+			SpaceLocationFlags::POSITION_VALID
+				| SpaceLocationFlags::POSITION_TRACKED
+				| SpaceLocationFlags::ORIENTATION_VALID
+				| SpaceLocationFlags::ORIENTATION_TRACKED,
+		);
+		tokio::task::spawn({
+			let handle = play_space.tracked_handle.clone();
+			async move {
+				handle.set_tracked(is_tracked);
+			}
+		});
+		if is_tracked {
+			play_space
+				.spatial
+				.set_local_transform(Mat4::from_rotation_translation(
+					location.pose.orientation.to_quat(),
+					location.pose.position.to_vec3(),
+				));
+		}
+	}
+	// session.reference_space_bounds_rect(openxr::ReferenceSpaceType::STAGE);
+
+	// if (World::has_bounds()
+	// 	&& World::get_bounds_size().x != 0.0
+	// 	&& World::get_bounds_size().y != 0.0)
+	// {
+	// 	let bounds = World::get_bounds_size();
+	// 	vec![
+	// 		((bounds.x).into(), (bounds.y).into()),
+	// 		((bounds.x).into(), (-bounds.y).into()),
+	// 		((-bounds.x).into(), (-bounds.y).into()),
+	// 		((-bounds.x).into(), (bounds.y).into()),
+	// 	]
+	// } else {
+	// 	vec![]
+	// }
+}
+
+#[derive(Resource)]
+pub struct PlaySpace {
+	spatial: Arc<Spatial>,
+	_spatial_handle: ObjectHandle<SpatialRef>,
+	tracked_handle: ObjectHandle<Tracked>,
+	bounds: Arc<RwLock<Vec<(f64, f64)>>>,
+}
+pub struct PlaySpaceBounds(Arc<RwLock<Vec<(f64, f64)>>>);
 impl PlaySpaceBounds {
-	pub async fn create(connection: &Connection) {
+	pub async fn create(connection: &Connection, data: Arc<RwLock<Vec<(f64, f64)>>>) {
 		connection
 			.object_server()
-			.at("/org/stardustxr/PlaySpace", Self)
+			.at("/org/stardustxr/PlaySpace", Self(data))
 			.await
 			.unwrap();
 	}
@@ -15,19 +154,6 @@ impl PlaySpaceBounds {
 impl PlaySpaceBounds {
 	#[zbus(property)]
 	fn bounds(&self) -> Vec<(f64, f64)> {
-		if (World::has_bounds()
+		self.0.read().clone()
-			&& World::get_bounds_size().x != 0.0
-			&& World::get_bounds_size().y != 0.0)
-		{
-			let bounds = World::get_bounds_size();
-			vec![
-				((bounds.x).into(), (bounds.y).into()),
-				((bounds.x).into(), (-bounds.y).into()),
-				((-bounds.x).into(), (-bounds.y).into()),
-				((-bounds.x).into(), (bounds.y).into()),
-			]
-		} else {
-			vec![]
-		}
 	}
 }