feat(input): hand

src/main.rs

@@ -7,6 +7,7 @@ use crate::core::destroy_queue;
 use crate::nodes::model::{MODELS_TO_DROP, MODEL_REGISTRY};
 use crate::nodes::{hmd, input};
 use crate::objects::input::mouse_pointer::MousePointer;
+use crate::objects::input::sk_hand::SkHand;
 use crate::wayland::Wayland;
 
 use self::core::eventloop::EventLoop;
@@ -14,6 +15,7 @@ use anyhow::Result;
 use clap::Parser;
 use slog::Drain;
 use std::sync::Arc;
+use stereokit::input::Handed;
 use stereokit::{lifecycle::DisplayMode, Settings};
 use tokio::{runtime::Handle, sync::oneshot};
 
@@ -49,8 +51,10 @@ fn main() -> Result<()> {
 	println!("Init StereoKit");
 
 	let mouse_pointer = cli_args.flatscreen.then(MousePointer::new);
+	let mut hands =
+		(!cli_args.flatscreen).then(|| [SkHand::new(Handed::Left), SkHand::new(Handed::Right)]);
 
-	if cli_args.flatscreen {
+	if hands.is_none() {
 		unsafe {
 			stereokit::sys::input_hand_visible(stereokit::sys::handed__handed_left, false as i32);
 			stereokit::sys::input_hand_visible(stereokit::sys::handed__handed_right, false as i32);
@@ -81,6 +85,10 @@ fn main() -> Result<()> {
 			if let Some(mouse_pointer) = &mouse_pointer {
 				mouse_pointer.update(&stereokit);
 			}
+			if let Some(hands) = &mut hands {
+				hands[0].update(&stereokit);
+				hands[1].update(&stereokit);
+			}
 			input::process_input();
 
 			wayland.make_context_current();

src/nodes/input/hand.rs

@@ -0,0 +1,84 @@
+use crate::nodes::fields::Field;
+use crate::nodes::spatial::Spatial;
+use glam::{vec3a, Mat4};
+use libstardustxr::schemas::input_hand::HandT;
+use libstardustxr::schemas::{common::JointT, input::InputDataRaw};
+use std::sync::Arc;
+
+use super::{DistanceLink, InputSpecialization};
+
+impl InputSpecialization for HandT {
+	fn distance(&self, space: &Arc<Spatial>, field: &Field) -> f32 {
+		let mut min_distance = f32::MAX;
+
+		for tip in [
+			&self.thumb.tip.position,
+			&self.index.tip.position,
+			&self.middle.tip.position,
+			&self.ring.tip.position,
+			&self.little.tip.position,
+		] {
+			min_distance = min_distance.min(field.distance(space, vec3a(tip.x, tip.y, tip.z)));
+		}
+
+		min_distance
+	}
+	fn serialize(
+		&self,
+		fbb: &mut flatbuffers::FlatBufferBuilder,
+		_distance_link: &DistanceLink,
+		local_to_handler_matrix: Mat4,
+	) -> (
+		InputDataRaw,
+		flatbuffers::WIPOffset<flatbuffers::UnionWIPOffset>,
+	) {
+		let mut hand = self.clone();
+		let mut joints: Vec<&mut JointT> = Vec::new();
+
+		joints.extend([&mut hand.palm, &mut hand.wrist]);
+		if let Some(elbow) = &mut hand.elbow {
+			joints.push(elbow);
+		}
+		for finger in [
+			&mut hand.index,
+			&mut hand.middle,
+			&mut hand.ring,
+			&mut hand.little,
+		] {
+			joints.extend([
+				&mut finger.tip,
+				&mut finger.distal,
+				&mut finger.intermediate,
+				&mut finger.proximal,
+				&mut finger.metacarpal,
+			]);
+		}
+		joints.extend([
+			&mut hand.thumb.tip,
+			&mut hand.thumb.distal,
+			&mut hand.thumb.proximal,
+			&mut hand.thumb.metacarpal,
+		]);
+
+		for joint in joints {
+			let rotation: mint::Quaternion<f32> = joint.rotation.clone().into();
+			let position: mint::Vector3<f32> = joint.position.clone().into();
+			let joint_matrix = Mat4::from_rotation_translation(rotation.into(), position.into())
+				* local_to_handler_matrix;
+			let (_, rotation, position) = joint_matrix.to_scale_rotation_translation();
+			let rotation: mint::Quaternion<f32> = rotation.into();
+			let position: mint::Vector3<f32> = position.into();
+			joint.position = position.into();
+			joint.rotation = rotation.into();
+		}
+
+		(InputDataRaw::Hand, hand.pack(fbb).as_union_value())
+	}
+	fn serialize_datamap(&self) -> Vec<u8> {
+		let mut fbb = flexbuffers::Builder::default();
+		let mut map = fbb.start_map();
+		map.push("right", self.right);
+		map.end_map();
+		fbb.view().to_vec()
+	}
+}

src/nodes/input/mod.rs

@@ -1,3 +1,4 @@
+pub mod hand;
 pub mod pointer;
 
 use self::pointer::Pointer;
@@ -11,7 +12,9 @@ use crate::core::registry::Registry;
 use anyhow::{anyhow, ensure, Result};
 use glam::Mat4;
 use libstardustxr::schemas::input::{InputData, InputDataArgs, InputDataRaw};
+use libstardustxr::schemas::input_hand::HandT;
 use nanoid::nanoid;
+use parking_lot::Mutex;
 use std::ops::Deref;
 use std::sync::atomic::Ordering;
 use std::sync::{Arc, Weak};
@@ -34,28 +37,32 @@ pub trait InputSpecialization: Send + Sync {
 }
 pub enum InputType {
 	Pointer(Pointer),
+	Hand(Box<HandT>),
 }
 impl Deref for InputType {
 	type Target = dyn InputSpecialization;
 	fn deref(&self) -> &Self::Target {
 		match self {
 			InputType::Pointer(p) => p,
+			InputType::Hand(h) => h.as_ref(),
 		}
 	}
 }
 
 pub struct InputMethod {
 	pub uid: String,
+	pub enabled: Mutex<bool>,
 	pub spatial: Arc<Spatial>,
-	pub specialization: InputType,
+	pub specialization: Mutex<InputType>,
 	pub captures: Registry<InputHandler>,
 }
 impl InputMethod {
 	pub fn new(spatial: Arc<Spatial>, specialization: InputType) -> Arc<InputMethod> {
 		let method = InputMethod {
 			uid: nanoid!(),
+			enabled: Mutex::new(true),
 			spatial,
-			specialization,
+			specialization: Mutex::new(specialization),
 			captures: Registry::new(),
 		};
 		INPUT_METHOD_REGISTRY.add(method)
@@ -69,8 +76,9 @@ impl InputMethod {
 
 		let method = InputMethod {
 			uid: node.uid.clone(),
+			enabled: Mutex::new(true),
 			spatial: node.spatial.get().unwrap().clone(),
-			specialization,
+			specialization: Mutex::new(specialization),
 			captures: Registry::new(),
 		};
 		let method = INPUT_METHOD_REGISTRY.add(method);
@@ -78,10 +86,10 @@ impl InputMethod {
 		Ok(())
 	}
 	fn distance(&self, to: &Field) -> f32 {
-		self.specialization.distance(&self.spatial, &to)
+		self.specialization.lock().distance(&self.spatial, to)
 	}
 	fn serialize_datamap(&self) -> Vec<u8> {
-		self.specialization.serialize_datamap()
+		self.specialization.lock().serialize_datamap()
 	}
 }
 impl Drop for InputMethod {
@@ -115,7 +123,7 @@ impl DistanceLink {
 		let uid = Some(fbb.create_string(&self.method.uid));
 		let datamap = Some(fbb.create_vector(datamap));
 
-		let (input_type, input_data) = self.method.specialization.serialize(
+		let (input_type, input_data) = self.method.specialization.lock().serialize(
 			&mut fbb,
 			self,
 			Spatial::space_to_space_matrix(Some(&self.method.spatial), Some(&self.handler.spatial)),
@@ -227,7 +235,11 @@ pub fn create_input_handler_flex(
 }
 
 pub fn process_input() {
-	for method in INPUT_METHOD_REGISTRY.get_valid_contents() {
+	for method in INPUT_METHOD_REGISTRY
+		.get_valid_contents()
+		.iter()
+		.filter(|method| *method.enabled.lock())
+	{
 		let mut distance_links: Vec<DistanceLink> = INPUT_HANDLER_REGISTRY
 			.get_valid_contents()
 			.into_iter()

src/nodes/input/pointer.rs

@@ -1,14 +1,12 @@
+use super::{DistanceLink, InputSpecialization};
+use crate::nodes::fields::{ray_march, Field, Ray, RayMarchResult};
+use crate::nodes::spatial::Spatial;
 use glam::{vec3, Mat4};
 use libstardustxr::schemas::input::InputDataRaw;
 use libstardustxr::schemas::input_pointer;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::Arc;
 
-use crate::nodes::fields::{ray_march, Field, Ray, RayMarchResult};
-use crate::nodes::spatial::Spatial;
-
-use super::{DistanceLink, InputSpecialization};
-
 #[derive(Default)]
 pub struct Pointer {
 	grab: AtomicBool,

src/objects/input/mod.rs

@@ -1 +1,2 @@
 pub mod mouse_pointer;
+pub mod sk_hand;

src/objects/input/sk_hand.rs

@@ -0,0 +1,72 @@
+use crate::nodes::{
+	input::{InputMethod, InputType},
+	spatial::Spatial,
+};
+use glam::Mat4;
+use libstardustxr::schemas::{common::JointT, input_hand::HandT};
+use std::sync::{Arc, Weak};
+use stereokit::{
+	input::{Handed, Joint as SkJoint},
+	StereoKit,
+};
+
+fn convert_joint(joint: SkJoint) -> JointT {
+	JointT {
+		position: joint.position.into(),
+		rotation: joint.orientation.into(),
+		radius: joint.radius,
+	}
+}
+
+pub struct SkHand {
+	hand: Arc<InputMethod>,
+	handed: Handed,
+}
+impl SkHand {
+	pub fn new(handed: Handed) -> Self {
+		let mut sk_hand = HandT::default();
+		sk_hand.right = handed == Handed::Right;
+		SkHand {
+			hand: InputMethod::new(
+				Spatial::new(Weak::new(), None, Mat4::IDENTITY),
+				InputType::Hand(Box::new(sk_hand)),
+			),
+			handed,
+		}
+	}
+	pub fn update(&mut self, sk: &StereoKit) {
+		if let InputType::Hand(hand) = &mut *self.hand.specialization.lock() {
+			let sk_hand = *sk.input_hand(self.handed);
+			// *self.hand.enabled.lock() = sk_hand.tracked_state.is_active();
+			// if sk_hand.tracked_state.is_active() {
+			hand.thumb.tip = convert_joint(sk_hand.fingers[0][4]);
+			hand.thumb.distal = convert_joint(sk_hand.fingers[0][3]);
+			hand.thumb.proximal = convert_joint(sk_hand.fingers[0][2]);
+			hand.thumb.metacarpal = convert_joint(sk_hand.fingers[0][1]);
+
+			for (finger, sk_finger) in [
+				(&mut hand.index, sk_hand.fingers[1]),
+				(&mut hand.middle, sk_hand.fingers[2]),
+				(&mut hand.ring, sk_hand.fingers[3]),
+				(&mut hand.little, sk_hand.fingers[4]),
+			] {
+				finger.tip = convert_joint(sk_finger[4]);
+				finger.distal = convert_joint(sk_finger[3]);
+				finger.intermediate = convert_joint(sk_finger[2]);
+				finger.proximal = convert_joint(sk_finger[1]);
+				finger.metacarpal = convert_joint(sk_finger[0]);
+			}
+
+			hand.palm.position = sk_hand.palm.position.into();
+			hand.palm.rotation = sk_hand.palm.orientation.into();
+			hand.palm.radius = (sk_hand.fingers[2][0].radius + sk_hand.fingers[2][1].radius) * 0.5;
+
+			hand.wrist.position = sk_hand.wrist.position.into();
+			hand.wrist.rotation = sk_hand.wrist.orientation.into();
+			hand.wrist.radius = (sk_hand.fingers[0][0].radius + sk_hand.fingers[4][0].radius) * 0.5;
+
+			hand.elbow = None;
+			// }
+		}
+	}
+}