feat: working handtracking input methodsRUST_LOG=info,naga=warn dbus-run-session cargo run -- -e ~/build/stardust/env.sh -o 300!

Signed-off-by: Schmarni <marnistromer@gmail.com>

src/objects/input/sk_hand.rs

@@ -1,15 +1,27 @@
 use crate::core::client::INTERNAL_CLIENT;
 use crate::nodes::OwnedNode;
 use crate::nodes::fields::{Field, FieldTrait};
-use crate::nodes::input::{INPUT_HANDLER_REGISTRY, InputDataType, InputHandler};
+use crate::nodes::input::{Finger, INPUT_HANDLER_REGISTRY, InputDataType, InputHandler, Thumb};
 use crate::nodes::{
 	Node,
 	input::{Hand, InputMethod, Joint},
 	spatial::Spatial,
 };
 use crate::objects::{ObjectHandle, SpatialRef, Tracked};
+use crate::{DbusConnection, ObjectRegistryRes, PreFrameWait};
+use bevy::prelude::Transform as BevyTransform;
+use bevy::prelude::*;
+use bevy_mod_openxr::helper_traits::{ToQuat, ToVec3};
+use bevy_mod_openxr::resources::OxrFrameState;
+use bevy_mod_openxr::session::OxrSession;
+use bevy_mod_xr::hands::{HandBone, HandSide, XrHandBoneEntities, XrHandBoneRadius};
+use bevy_mod_xr::session::{XrPreDestroySession, XrSessionCreated};
+use bevy_mod_xr::spaces::{XrPrimaryReferenceSpace, XrSpaceLocationFlags};
+use bevy_sk::hand::GRADIENT_TEXTURE_HANDLE;
+use bevy_sk::vr_materials::PbrMaterial;
 use color_eyre::eyre::Result;
 use glam::{Mat4, Quat, Vec3};
+use openxr::{HandJointLocation, SpaceLocationFlags};
 use serde::{Deserialize, Serialize};
 use stardust_xr::values::Datamap;
 use std::sync::Arc;
@@ -21,15 +33,146 @@ use zbus::Connection;
 
 use super::{CaptureManager, get_sorted_handlers};
 
-fn convert_joint(joint: HandJoint) -> Joint {
+pub struct HandPlugin;
+impl Plugin for HandPlugin {
+	fn build(&self, app: &mut App) {
+		app.add_systems(PreFrameWait, update_hands);
+		app.add_systems(XrSessionCreated, create_trackers);
+		app.add_systems(XrPreDestroySession, destroy_trackers);
+		app.add_systems(PostUpdate, update_hand_material);
+		app.add_systems(Startup, setup);
+	}
+}
+fn update_hands(
+	mut hands: ResMut<Hands>,
+	session: Option<Res<OxrSession>>,
+	state: Option<Res<OxrFrameState>>,
+	ref_space: Option<Res<XrPrimaryReferenceSpace>>,
+	mut materials: ResMut<Assets<PbrMaterial>>,
+	mut joint_query: Query<(
+		&mut BevyTransform,
+		&mut XrSpaceLocationFlags,
+		&mut XrHandBoneRadius,
+	)>,
+	joints_query: Query<&XrHandBoneEntities>,
+) {
+	let (Some(session), Some(state), Some(ref_space)) = (session, state, ref_space) else {
+		tokio::task::spawn({
+			let left = hands.left.tracked.clone();
+			let right = hands.right.tracked.clone();
+			async move {
+				left.set_tracked(false);
+				right.set_tracked(false);
+			}
+		});
+		return;
+	};
+	let get_joints = |hand: &mut SkHand| -> Option<openxr::HandJointLocations> {
+		let Some(tracker) = hand.tracker.as_ref() else {
+			hand.input.spatial.node().unwrap().set_enabled(false);
+			let handle = hand.tracked.clone();
+			tokio::task::spawn(async move {
+				handle.set_tracked(false);
+			});
+			return None;
+		};
+		// this won't be correct with pipelined rendering
+		session
+			.locate_hand_joints(tracker, &ref_space, state.predicted_display_time)
+			.inspect_err(|err| error!("Error while locating hand joints"))
+			.ok()
+			.flatten()
+	};
+	let joints_left = get_joints(&mut hands.left);
+	let joints_right = get_joints(&mut hands.right);
+	hands.left.update(joints_left.as_ref(), &mut materials);
+	hands.right.update(joints_right.as_ref(), &mut materials);
+}
+
+fn pinch_between(joint_1: &Joint, joint_2: &Joint) -> f32 {
+	const PINCH_MAX: f32 = 0.11;
+	const PINCH_ACTIVACTION_DISTANCE: f32 = 0.01;
+	let combined_radius = joint_1.radius + joint_2.radius;
+	let pinch_dist =
+		Vec3::from(joint_1.position).distance(Vec3::from(joint_2.position)) - combined_radius;
+	(1.0 - ((pinch_dist - PINCH_ACTIVACTION_DISTANCE) / (PINCH_MAX - PINCH_ACTIVACTION_DISTANCE)))
+		.clamp(0.0, 1.0)
+}
+
+fn create_trackers(session: Res<OxrSession>, mut hands: ResMut<Hands>) {
+	hands.left.tracker = session
+		.create_hand_tracker(openxr::HandEXT::LEFT)
+		.inspect_err(|err| error!("failed to create left hand tracker"))
+		.ok();
+	hands.right.tracker = session
+		.create_hand_tracker(openxr::HandEXT::RIGHT)
+		.inspect_err(|err| error!("failed to create right hand tracker"))
+		.ok();
+}
+fn destroy_trackers(mut hands: ResMut<Hands>) {
+	hands.left.tracker.take();
+	hands.right.tracker.take();
+}
+#[derive(Component)]
+struct CorrectHandMaterial;
+fn update_hand_material(
+	query: Query<
+		(Entity, &HandSide),
+		(
+			With<XrHandBoneEntities>,
+			With<MeshMaterial3d<PbrMaterial>>,
+			Without<CorrectHandMaterial>,
+		),
+	>,
+	mut cmds: Commands,
+	hands: Res<Hands>,
+) {
+	for (entity, side) in &query {
+		let handle = match side {
+			HandSide::Left => hands.left.material.clone(),
+			HandSide::Right => hands.right.material.clone(),
+		};
+		cmds.entity(entity)
+			.insert(MeshMaterial3d(handle))
+			.insert(CorrectHandMaterial);
+	}
+}
+
+fn setup(
+	connection: Res<DbusConnection>,
+	mut cmds: Commands,
+	mut materials: ResMut<Assets<PbrMaterial>>,
+) {
+	tokio::task::spawn({
+		let connection = connection.clone();
+		async move {
+			connection
+				.request_name("org.stardustxr.Hands")
+				.await
+				.unwrap();
+		}
+	});
+	cmds.insert_resource(Hands {
+		left: SkHand::new(&connection, HandSide::Left, &mut materials).unwrap(),
+		right: SkHand::new(&connection, HandSide::Right, &mut materials).unwrap(),
+	});
+}
+
+fn convert_joint(joint: HandJointLocation) -> Joint {
 	Joint {
-		position: Vec3::from(joint.position).into(),
-		rotation: Quat::from(joint.orientation).into(),
+		position: joint.pose.position.to_vec3().into(),
+		rotation: joint.pose.orientation.to_quat().into(),
 		radius: joint.radius,
 		distance: 0.0,
 	}
 }
 
+#[derive(Resource)]
+struct Hands {
+	left: SkHand,
+	right: SkHand,
+}
+
 #[derive(Default, Deserialize, Serialize)]
 struct HandDatamap {
 	pinch_strength: f32,
@@ -40,117 +183,158 @@ pub struct SkHand {
 	_node: OwnedNode,
 	palm_spatial: Arc<Spatial>,
 	palm_object: ObjectHandle<SpatialRef>,
-	handed: Handed,
+	side: HandSide,
 	input: Arc<InputMethod>,
 	capture_manager: CaptureManager,
 	datamap: HandDatamap,
 	tracked: ObjectHandle<Tracked>,
+	tracker: Option<openxr::HandTracker>,
+	captured: bool,
+	material: Handle<PbrMaterial>,
 }
 impl SkHand {
-	pub fn new(connection: &Connection, handed: Handed) -> Result<Self> {
+	pub fn new(
+		connection: &Connection,
+		side: HandSide,
+		materials: &mut Assets<PbrMaterial>,
+	) -> Result<Self> {
 		let (palm_spatial, palm_object) = SpatialRef::create(
 			connection,
 			&("/org/stardustxr/Hand/".to_string()
-				+ match handed {
-					Handed::Left => "left",
-					_ => "right",
+				+ match side {
+					HandSide::Left => "left",
+					HandSide::Right => "right",
 				} + "/palm"),
 		);
 		let tracked = Tracked::new(
 			connection,
 			&("/org/stardustxr/Hand/".to_string()
-				+ match handed {
-					Handed::Left => "left",
-					_ => "right",
+				+ match side {
+					HandSide::Left => "left",
+					HandSide::Right => "right",
 				}),
 		);
-		let _node = Node::generate(&INTERNAL_CLIENT, false).add_to_scenegraph_owned()?;
-		Spatial::add_to(&_node.0, None, Mat4::IDENTITY, false);
+		let node = Node::generate(&INTERNAL_CLIENT, false).add_to_scenegraph_owned()?;
+		Spatial::add_to(&node.0, None, Mat4::IDENTITY, false);
 		let hand = InputDataType::Hand(Hand {
-			right: handed == Handed::Right,
+			right: matches!(side, HandSide::Right),
 			..Default::default()
 		});
 		let datamap = Datamap::from_typed(HandDatamap::default())?;
-		let input = InputMethod::add_to(&_node.0, hand, datamap)?;
-		Input::hand_visible(handed, true);
+		let input = InputMethod::add_to(&node.0, hand, datamap)?;
 
+		let material = materials.add(PbrMaterial {
+			color: Srgba::new(1.0, 1.0, 1.0, 1.0).into(),
+			alpha_mode: AlphaMode::Blend,
+			use_stereokit_uvs: false,
+			diffuse_texture: Some(GRADIENT_TEXTURE_HANDLE),
+			roughness: 1.0,
+			..default()
+		});
 		Ok(SkHand {
-			_node,
+			_node: node,
 			palm_spatial,
 			palm_object,
-			handed,
+			side,
 			input,
 			tracked,
 			capture_manager: CaptureManager::default(),
 			datamap: Default::default(),
+			tracker: None,
+			material,
+			captured: false,
 		})
 	}
-	pub fn update(&mut self, sk: &Sk, token: &MainThreadToken, material: &mut Material) {
-		let sk_hand = Input::hand(self.handed);
-		let real_hand = Input::hand_source(self.handed) as u32 == HandSource::Articulated as u32;
-		if let InputDataType::Hand(hand) = &mut *self.input.data.lock() {
-			let input_node = self.input.spatial.node().unwrap();
-			input_node.set_enabled(
-				(real_hand || sk.get_active_display_mode() == DisplayMode::Flatscreen)
-					&& sk_hand.tracked.is_active(),
-			);
-			let enabled = input_node.enabled();
-			tokio::spawn({
-				// this is suboptimal since it probably allocates a fresh string every frame
-				let handle = self.tracked.clone();
-				async move {
-					handle.set_tracked(enabled).await;
-				}
+	fn update(
+		&mut self,
+		joints: Option<&openxr::HandJointLocations>,
+		materials: &mut ResMut<Assets<PbrMaterial>>,
+	) {
+		// TODO: use the hand data source ext
+		let real_hand = true;
+		let input_node = self.input.spatial.node().unwrap();
+		let is_tracked = real_hand
+			&& joints.is_some_and(|v| {
+				v.iter().all(|v| {
+					v.location_flags.contains(
+						SpaceLocationFlags::POSITION_VALID
+							| SpaceLocationFlags::POSITION_TRACKED
+							| SpaceLocationFlags::ORIENTATION_VALID
+							| SpaceLocationFlags::ORIENTATION_TRACKED,
+					)
+				})
 			});
-			if enabled {
-				hand.thumb.tip = convert_joint(sk_hand.fingers[0][4]);
-				hand.thumb.distal = convert_joint(sk_hand.fingers[0][3]);
-				hand.thumb.proximal = convert_joint(sk_hand.fingers[0][2]);
-				hand.thumb.metacarpal = convert_joint(sk_hand.fingers[0][1]);
-
-				for (finger, mut sk_finger) in [
-					(&mut hand.index, sk_hand.fingers[1]),
-					(&mut hand.middle, sk_hand.fingers[2]),
-					(&mut hand.ring, sk_hand.fingers[3]),
-					(&mut hand.little, sk_hand.fingers[4]),
-				] {
-					sk_finger[4].radius = 0.0;
-					finger.tip = convert_joint(sk_finger[4]);
-					finger.distal = convert_joint(sk_finger[3]);
-					finger.intermediate = convert_joint(sk_finger[2]);
-					finger.proximal = convert_joint(sk_finger[1]);
-					finger.metacarpal = convert_joint(sk_finger[0]);
-				}
-
-				hand.palm.position = Vec3::from(sk_hand.palm.position).into();
-				hand.palm.rotation = Quat::from(sk_hand.palm.orientation).into();
-				hand.palm.radius =
-					(sk_hand.fingers[2][0].radius + sk_hand.fingers[2][1].radius) * 0.5;
-
-				self.palm_spatial
-					.set_local_transform(Mat4::from_rotation_translation(
-						hand.palm.rotation.into(),
-						hand.palm.position.into(),
-					));
-
-				hand.wrist.position = Vec3::from(sk_hand.wrist.position).into();
-				hand.wrist.rotation = Quat::from(sk_hand.wrist.orientation).into();
-				hand.wrist.radius =
-					(sk_hand.fingers[0][0].radius + sk_hand.fingers[4][0].radius) * 0.5;
-
-				hand.elbow = None;
-
-				let hand_color = if self.capture_manager.capture.upgrade().is_none() {
-					Color128::new_rgb(1.0, 1.0, 1.0)
-				} else {
-					Color128::new_rgb(0.0, 1.0, 0.75)
-				};
-				material.color_tint(hand_color);
+		input_node.set_enabled(is_tracked);
+		tokio::task::spawn({
+			let handle = self.tracked.clone();
+			async move {
+				handle.set_tracked(is_tracked);
 			}
+		});
+		if is_tracked {
+			// cannot ever crash, is_tracked is only true of joints is some
+			let joints = joints.unwrap();
+			let new_hand = Hand {
+				right: matches!(self.side, HandSide::Right),
+				thumb: Thumb {
+					tip: convert_joint(joints[HandBone::ThumbTip as usize]),
+					distal: convert_joint(joints[HandBone::ThumbDistal as usize]),
+					proximal: convert_joint(joints[HandBone::ThumbProximal as usize]),
+					metacarpal: convert_joint(joints[HandBone::ThumbMetacarpal as usize]),
+				},
+				index: Finger {
+					tip: convert_joint(joints[HandBone::IndexTip as usize]),
+					distal: convert_joint(joints[HandBone::IndexDistal as usize]),
+					intermediate: convert_joint(joints[HandBone::IndexIntermediate as usize]),
+					proximal: convert_joint(joints[HandBone::IndexProximal as usize]),
+					metacarpal: convert_joint(joints[HandBone::IndexMetacarpal as usize]),
+				},
+				middle: Finger {
+					tip: convert_joint(joints[HandBone::MiddleTip as usize]),
+					distal: convert_joint(joints[HandBone::MiddleDistal as usize]),
+					intermediate: convert_joint(joints[HandBone::MiddleIntermediate as usize]),
+					proximal: convert_joint(joints[HandBone::MiddleProximal as usize]),
+					metacarpal: convert_joint(joints[HandBone::MiddleMetacarpal as usize]),
+				},
+				ring: Finger {
+					tip: convert_joint(joints[HandBone::RingTip as usize]),
+					distal: convert_joint(joints[HandBone::RingDistal as usize]),
+					intermediate: convert_joint(joints[HandBone::RingIntermediate as usize]),
+					proximal: convert_joint(joints[HandBone::RingProximal as usize]),
+					metacarpal: convert_joint(joints[HandBone::RingMetacarpal as usize]),
+				},
+				little: Finger {
+					tip: convert_joint(joints[HandBone::LittleTip as usize]),
+					distal: convert_joint(joints[HandBone::LittleDistal as usize]),
+					intermediate: convert_joint(joints[HandBone::LittleIntermediate as usize]),
+					proximal: convert_joint(joints[HandBone::LittleProximal as usize]),
+					metacarpal: convert_joint(joints[HandBone::LittleMetacarpal as usize]),
+				},
+				palm: convert_joint(joints[HandBone::Palm as usize]),
+				wrist: convert_joint(joints[HandBone::Wrist as usize]),
+				elbow: None,
+			};
+			self.palm_spatial
+				.set_local_transform(Mat4::from_rotation_translation(
+					new_hand.palm.rotation.into(),
+					new_hand.palm.position.into(),
+				));
+
+			self.datamap.pinch_strength = pinch_between(&new_hand.thumb.tip, &new_hand.index.tip);
+			// this is how stereokit calculates grab
+			self.datamap.grab_strength =
+				pinch_between(&new_hand.ring.tip, &new_hand.ring.metacarpal);
+
+			*self.input.data.lock() = InputDataType::Hand(new_hand);
+			*self.input.datamap.lock() = Datamap::from_typed(&self.datamap).unwrap();
+			let captured = self.capture_manager.capture.upgrade().is_some();
+			if captured && !self.captured {
+				materials.get_mut(&self.material).unwrap().color = Srgba::rgb(0., 1., 0.75).into();
+			} else if self.captured && !captured {
+				materials.get_mut(&self.material).unwrap().color = Srgba::rgb(1., 1.0, 1.0).into();
+			}
+			self.captured = captured;
 		}
-		self.datamap.pinch_strength = sk_hand.pinch_activation;
-		self.datamap.grab_strength = sk_hand.grip_activation;
-		*self.input.datamap.lock() = Datamap::from_typed(&self.datamap).unwrap();
 
 		let distance_calculator = |space: &Arc<Spatial>, data: &InputDataType, field: &Field| {
 			let InputDataType::Hand(hand) = data else {
@@ -183,11 +367,6 @@ impl SkHand {
 			.set_handler_order(sorted_handlers.iter().map(|(handler, _)| handler));
 	}
 }
-impl Drop for SkHand {
-	fn drop(&mut self) {
-		Input::hand_visible(self.handed, false);
-	}
-}
 
 fn joint_to_line_point(joint: &Joint, color: Color128) -> LinePoint {
 	LinePoint {

src/objects/mod.rs

@@ -13,6 +13,7 @@ use input::{
 	eye_pointer::EyePointer, mouse_pointer::MousePointer, sk_controller::SkController,
 	sk_hand::SkHand,
 };
+use parking_lot::RwLock;
 use play_space::PlaySpaceBounds;
 use stardust_xr::schemas::dbus::object_registry::ObjectRegistry;
 use std::{
@@ -34,16 +35,16 @@ enum Inputs {
 	XR {
 		controller_left: SkController,
 		controller_right: SkController,
-		hand_left: SkHand,
-		hand_right: SkHand,
+		// hand_left: SkHand,
+		// hand_right: SkHand,
 		eye_pointer: Option<EyePointer>,
 	},
 	MousePointer(MousePointer),
 	// Controllers((SkController, SkController)),
-	Hands {
-		left: SkHand,
-		right: SkHand,
-	},
+	// Hands {
+	// 	left: SkHand,
+	// 	right: SkHand,
+	// },
 }
 
 pub struct ServerObjects {
@@ -69,7 +70,8 @@ impl ServerObjects {
 		if play_space.is_some() {
 			let dbus_connection = connection.clone();
 			tokio::task::spawn(async move {
-				PlaySpaceBounds::create(&dbus_connection).await;
+				let play_space_data = Arc::new(RwLock::default());
+				PlaySpaceBounds::create(&dbus_connection, play_space_data).await;
 				dbus_connection
 					.request_name("org.stardustxr.PlaySpace")
 					.await
@@ -95,8 +97,8 @@ impl ServerObjects {
 			Inputs::XR {
 				controller_left: SkController::new(&connection, Handed::Left).unwrap(),
 				controller_right: SkController::new(&connection, Handed::Right).unwrap(),
-				hand_left: SkHand::new(&connection, Handed::Left).unwrap(),
-				hand_right: SkHand::new(&connection, Handed::Right).unwrap(),
+				// hand_left: SkHand::new(&connection, Handed::Left).unwrap(),
+				// hand_right: SkHand::new(&connection, Handed::Right).unwrap(),
 				eye_pointer: Device::has_eye_gaze()
 					.then(EyePointer::new)
 					.transpose()
@@ -166,8 +168,6 @@ impl ServerObjects {
 			Inputs::XR {
 				controller_left,
 				controller_right,
-				hand_left,
-				hand_right,
 				eye_pointer,
 			} => {
 				if !self.disable_controllers {
@@ -176,10 +176,10 @@ impl ServerObjects {
 				}
 				Input::hand_visible(Handed::Left, !self.disable_hands);
 				Input::hand_visible(Handed::Right, !self.disable_hands);
-				if !self.disable_hands {
-					hand_left.update(sk, token, &mut self.hand_materials[0]);
-					hand_right.update(sk, token, &mut self.hand_materials[1]);
-				}
+				// if !self.disable_hands {
+				// 	hand_left.update(sk, token, &mut self.hand_materials[0]);
+				// 	hand_right.update(sk, token, &mut self.hand_materials[1]);
+				// }
 				if let Some(eye_pointer) = eye_pointer {
 					eye_pointer.update();
 				}
@@ -191,10 +191,10 @@ impl ServerObjects {
 			// 	left.update(token);
 			// 	right.update(token);
 			// }
-			Inputs::Hands { left, right } => {
-				left.update(sk, token, &mut self.hand_materials[0]);
-				right.update(sk, token, &mut self.hand_materials[1]);
-			}
+			// Inputs::Hands { left, right } => {
+			// 	left.update(sk, token, &mut self.hand_materials[0]);
+			// 	right.update(sk, token, &mut self.hand_materials[1]);
+			// }
 		}
 	}
 }

src/objects/play_space.rs

@@ -1,12 +1,151 @@
+use std::sync::Arc;
+
+use bevy::prelude::*;
+use bevy_mod_openxr::{
+	helper_traits::{ToQuat, ToVec3},
+	resources::OxrFrameState,
+	session::OxrSession,
+};
+use bevy_mod_xr::{
+	session::{XrPreDestroySession, XrSessionCreated},
+	spaces::{XrPrimaryReferenceSpace, XrReferenceSpace, XrSpace},
+};
+use openxr::SpaceLocationFlags;
+use parking_lot::RwLock;
 use stereokit_rust::system::World;
 use zbus::{Connection, ObjectServer, interface};
 
-pub struct PlaySpaceBounds;
+use crate::{DbusConnection, PreFrameWait, nodes::spatial::Spatial};
+
+use super::{ObjectHandle, SpatialRef, Tracked};
+
+pub struct PlaySpacePlugin;
+impl Plugin for PlaySpacePlugin {
+	fn build(&self, app: &mut App) {
+		app.add_systems(XrPreDestroySession, destroy_stage_space);
+		app.add_systems(XrSessionCreated, create_stage_space);
+		app.add_systems(PreFrameWait, update);
+		app.add_systems(Startup, setup);
+	}
+}
+
+fn setup(connection: Res<DbusConnection>, mut cmds: Commands) {
+	let (spatial, spatial_handle) = SpatialRef::create(&connection, "/org/stardustxr/PlaySpace");
+	// the OpenXR session might not exist quite yet
+	let tracked = Tracked::new(&connection, "/org/stardustxr/PlaySpace");
+	let dbus_connection = connection.clone();
+	let play_space_data = Arc::new(RwLock::default());
+	tokio::task::spawn({
+		let data = play_space_data.clone();
+		async move {
+			PlaySpaceBounds::create(&dbus_connection, data).await;
+			dbus_connection
+				.request_name("org.stardustxr.PlaySpace")
+				.await
+				.unwrap();
+		}
+	});
+	cmds.insert_resource(PlaySpace {
+		spatial,
+		_spatial_handle: spatial_handle,
+		tracked_handle: tracked,
+		bounds: play_space_data,
+	});
+}
+
+#[derive(Resource)]
+struct StageSpace(XrSpace);
+fn create_stage_space(session: Res<OxrSession>, mut cmds: Commands) {
+	let space = session
+		.create_reference_space(openxr::ReferenceSpaceType::STAGE, Transform::IDENTITY)
+		.inspect_err(|err| error!("failed to create Stage XrSpace"))
+		.ok();
+	if let Some(space) = space {
+		cmds.insert_resource(StageSpace(space.0));
+	}
+}
+fn destroy_stage_space(session: Res<OxrSession>, mut cmds: Commands, stage: Res<StageSpace>) {
+	session.destroy_space(stage.0);
+	cmds.remove_resource::<StageSpace>();
+}
+
+/// TODO: impl this
+fn update(
+	session: Option<Res<OxrSession>>,
+	stage: Option<Res<StageSpace>>,
+	ref_space: Option<Res<XrPrimaryReferenceSpace>>,
+	play_space: Res<PlaySpace>,
+	state: Option<Res<OxrFrameState>>,
+) {
+	let (Some(session), Some(stage), Some(ref_space), Some(state)) =
+		(session, stage, ref_space, state)
+	else {
+		play_space.bounds.write().drain(..);
+		tokio::task::spawn({
+			let handle = play_space.tracked_handle.clone();
+			async move {
+				handle.set_tracked(false);
+			}
+		});
+		return;
+	};
+	// this won't be correct with pipelined rendering
+	let location = session
+		.locate_space(&stage.0, &ref_space, state.predicted_display_time)
+		.inspect_err(|err| error!("Error while Locating OpenXR Stage Space {err}"));
+	if let Ok(location) = location {
+		let is_tracked = location.location_flags.contains(
+			SpaceLocationFlags::POSITION_VALID
+				| SpaceLocationFlags::POSITION_TRACKED
+				| SpaceLocationFlags::ORIENTATION_VALID
+				| SpaceLocationFlags::ORIENTATION_TRACKED,
+		);
+		tokio::task::spawn({
+			let handle = play_space.tracked_handle.clone();
+			async move {
+				handle.set_tracked(is_tracked);
+			}
+		});
+		if is_tracked {
+			play_space
+				.spatial
+				.set_local_transform(Mat4::from_rotation_translation(
+					location.pose.orientation.to_quat(),
+					location.pose.position.to_vec3(),
+				));
+		}
+	}
+	// session.reference_space_bounds_rect(openxr::ReferenceSpaceType::STAGE);
+
+	// if (World::has_bounds()
+	// 	&& World::get_bounds_size().x != 0.0
+	// 	&& World::get_bounds_size().y != 0.0)
+	// {
+	// 	let bounds = World::get_bounds_size();
+	// 	vec![
+	// 		((bounds.x).into(), (bounds.y).into()),
+	// 		((bounds.x).into(), (-bounds.y).into()),
+	// 		((-bounds.x).into(), (-bounds.y).into()),
+	// 		((-bounds.x).into(), (bounds.y).into()),
+	// 	]
+	// } else {
+	// 	vec![]
+	// }
+}
+
+#[derive(Resource)]
+pub struct PlaySpace {
+	spatial: Arc<Spatial>,
+	_spatial_handle: ObjectHandle<SpatialRef>,
+	tracked_handle: ObjectHandle<Tracked>,
+	bounds: Arc<RwLock<Vec<(f64, f64)>>>,
+}
+pub struct PlaySpaceBounds(Arc<RwLock<Vec<(f64, f64)>>>);
 impl PlaySpaceBounds {
-	pub async fn create(connection: &Connection) {
+	pub async fn create(connection: &Connection, data: Arc<RwLock<Vec<(f64, f64)>>>) {
 		connection
 			.object_server()
-			.at("/org/stardustxr/PlaySpace", Self)
+			.at("/org/stardustxr/PlaySpace", Self(data))
 			.await
 			.unwrap();
 	}
@@ -15,19 +154,6 @@ impl PlaySpaceBounds {
 impl PlaySpaceBounds {
 	#[zbus(property)]
 	fn bounds(&self) -> Vec<(f64, f64)> {
-		if (World::has_bounds()
-			&& World::get_bounds_size().x != 0.0
-			&& World::get_bounds_size().y != 0.0)
-		{
-			let bounds = World::get_bounds_size();
-			vec![
-				((bounds.x).into(), (bounds.y).into()),
-				((bounds.x).into(), (-bounds.y).into()),
-				((-bounds.x).into(), (-bounds.y).into()),
-				((-bounds.x).into(), (bounds.y).into()),
-			]
-		} else {
-			vec![]
-		}
+		self.0.read().clone()
 	}
 }