From ee250b33fa054c49d2704f994acb0511ed716829 Mon Sep 17 00:00:00 2001 From: Nova Date: Sat, 17 Sep 2022 17:26:50 -0400 Subject: [PATCH] feat(input): hand --- src/main.rs | 10 ++++- src/nodes/input/hand.rs | 84 ++++++++++++++++++++++++++++++++++++ src/nodes/input/mod.rs | 26 ++++++++--- src/nodes/input/pointer.rs | 8 ++-- src/objects/input/mod.rs | 1 + src/objects/input/sk_hand.rs | 72 +++++++++++++++++++++++++++++++ 6 files changed, 188 insertions(+), 13 deletions(-) create mode 100644 src/nodes/input/hand.rs create mode 100644 src/objects/input/sk_hand.rs diff --git a/src/main.rs b/src/main.rs index cddf69d..d688811 100644 --- a/src/main.rs +++ b/src/main.rs @@ -7,6 +7,7 @@ use crate::core::destroy_queue; use crate::nodes::model::{MODELS_TO_DROP, MODEL_REGISTRY}; use crate::nodes::{hmd, input}; use crate::objects::input::mouse_pointer::MousePointer; +use crate::objects::input::sk_hand::SkHand; use crate::wayland::Wayland; use self::core::eventloop::EventLoop; @@ -14,6 +15,7 @@ use anyhow::Result; use clap::Parser; use slog::Drain; use std::sync::Arc; +use stereokit::input::Handed; use stereokit::{lifecycle::DisplayMode, Settings}; use tokio::{runtime::Handle, sync::oneshot}; @@ -49,8 +51,10 @@ fn main() -> Result<()> { println!("Init StereoKit"); let mouse_pointer = cli_args.flatscreen.then(MousePointer::new); + let mut hands = + (!cli_args.flatscreen).then(|| [SkHand::new(Handed::Left), SkHand::new(Handed::Right)]); - if cli_args.flatscreen { + if hands.is_none() { unsafe { stereokit::sys::input_hand_visible(stereokit::sys::handed__handed_left, false as i32); stereokit::sys::input_hand_visible(stereokit::sys::handed__handed_right, false as i32); @@ -81,6 +85,10 @@ fn main() -> Result<()> { if let Some(mouse_pointer) = &mouse_pointer { mouse_pointer.update(&stereokit); } + if let Some(hands) = &mut hands { + hands[0].update(&stereokit); + hands[1].update(&stereokit); + } input::process_input(); wayland.make_context_current(); diff --git a/src/nodes/input/hand.rs b/src/nodes/input/hand.rs new file mode 100644 index 0000000..b60988f --- /dev/null +++ b/src/nodes/input/hand.rs @@ -0,0 +1,84 @@ +use crate::nodes::fields::Field; +use crate::nodes::spatial::Spatial; +use glam::{vec3a, Mat4}; +use libstardustxr::schemas::input_hand::HandT; +use libstardustxr::schemas::{common::JointT, input::InputDataRaw}; +use std::sync::Arc; + +use super::{DistanceLink, InputSpecialization}; + +impl InputSpecialization for HandT { + fn distance(&self, space: &Arc, field: &Field) -> f32 { + let mut min_distance = f32::MAX; + + for tip in [ + &self.thumb.tip.position, + &self.index.tip.position, + &self.middle.tip.position, + &self.ring.tip.position, + &self.little.tip.position, + ] { + min_distance = min_distance.min(field.distance(space, vec3a(tip.x, tip.y, tip.z))); + } + + min_distance + } + fn serialize( + &self, + fbb: &mut flatbuffers::FlatBufferBuilder, + _distance_link: &DistanceLink, + local_to_handler_matrix: Mat4, + ) -> ( + InputDataRaw, + flatbuffers::WIPOffset, + ) { + let mut hand = self.clone(); + let mut joints: Vec<&mut JointT> = Vec::new(); + + joints.extend([&mut hand.palm, &mut hand.wrist]); + if let Some(elbow) = &mut hand.elbow { + joints.push(elbow); + } + for finger in [ + &mut hand.index, + &mut hand.middle, + &mut hand.ring, + &mut hand.little, + ] { + joints.extend([ + &mut finger.tip, + &mut finger.distal, + &mut finger.intermediate, + &mut finger.proximal, + &mut finger.metacarpal, + ]); + } + joints.extend([ + &mut hand.thumb.tip, + &mut hand.thumb.distal, + &mut hand.thumb.proximal, + &mut hand.thumb.metacarpal, + ]); + + for joint in joints { + let rotation: mint::Quaternion = joint.rotation.clone().into(); + let position: mint::Vector3 = joint.position.clone().into(); + let joint_matrix = Mat4::from_rotation_translation(rotation.into(), position.into()) + * local_to_handler_matrix; + let (_, rotation, position) = joint_matrix.to_scale_rotation_translation(); + let rotation: mint::Quaternion = rotation.into(); + let position: mint::Vector3 = position.into(); + joint.position = position.into(); + joint.rotation = rotation.into(); + } + + (InputDataRaw::Hand, hand.pack(fbb).as_union_value()) + } + fn serialize_datamap(&self) -> Vec { + let mut fbb = flexbuffers::Builder::default(); + let mut map = fbb.start_map(); + map.push("right", self.right); + map.end_map(); + fbb.view().to_vec() + } +} diff --git a/src/nodes/input/mod.rs b/src/nodes/input/mod.rs index 1556a1d..337c706 100644 --- a/src/nodes/input/mod.rs +++ b/src/nodes/input/mod.rs @@ -1,3 +1,4 @@ +pub mod hand; pub mod pointer; use self::pointer::Pointer; @@ -11,7 +12,9 @@ use crate::core::registry::Registry; use anyhow::{anyhow, ensure, Result}; use glam::Mat4; use libstardustxr::schemas::input::{InputData, InputDataArgs, InputDataRaw}; +use libstardustxr::schemas::input_hand::HandT; use nanoid::nanoid; +use parking_lot::Mutex; use std::ops::Deref; use std::sync::atomic::Ordering; use std::sync::{Arc, Weak}; @@ -34,28 +37,32 @@ pub trait InputSpecialization: Send + Sync { } pub enum InputType { Pointer(Pointer), + Hand(Box), } impl Deref for InputType { type Target = dyn InputSpecialization; fn deref(&self) -> &Self::Target { match self { InputType::Pointer(p) => p, + InputType::Hand(h) => h.as_ref(), } } } pub struct InputMethod { pub uid: String, + pub enabled: Mutex, pub spatial: Arc, - pub specialization: InputType, + pub specialization: Mutex, pub captures: Registry, } impl InputMethod { pub fn new(spatial: Arc, specialization: InputType) -> Arc { let method = InputMethod { uid: nanoid!(), + enabled: Mutex::new(true), spatial, - specialization, + specialization: Mutex::new(specialization), captures: Registry::new(), }; INPUT_METHOD_REGISTRY.add(method) @@ -69,8 +76,9 @@ impl InputMethod { let method = InputMethod { uid: node.uid.clone(), + enabled: Mutex::new(true), spatial: node.spatial.get().unwrap().clone(), - specialization, + specialization: Mutex::new(specialization), captures: Registry::new(), }; let method = INPUT_METHOD_REGISTRY.add(method); @@ -78,10 +86,10 @@ impl InputMethod { Ok(()) } fn distance(&self, to: &Field) -> f32 { - self.specialization.distance(&self.spatial, &to) + self.specialization.lock().distance(&self.spatial, to) } fn serialize_datamap(&self) -> Vec { - self.specialization.serialize_datamap() + self.specialization.lock().serialize_datamap() } } impl Drop for InputMethod { @@ -115,7 +123,7 @@ impl DistanceLink { let uid = Some(fbb.create_string(&self.method.uid)); let datamap = Some(fbb.create_vector(datamap)); - let (input_type, input_data) = self.method.specialization.serialize( + let (input_type, input_data) = self.method.specialization.lock().serialize( &mut fbb, self, Spatial::space_to_space_matrix(Some(&self.method.spatial), Some(&self.handler.spatial)), @@ -227,7 +235,11 @@ pub fn create_input_handler_flex( } pub fn process_input() { - for method in INPUT_METHOD_REGISTRY.get_valid_contents() { + for method in INPUT_METHOD_REGISTRY + .get_valid_contents() + .iter() + .filter(|method| *method.enabled.lock()) + { let mut distance_links: Vec = INPUT_HANDLER_REGISTRY .get_valid_contents() .into_iter() diff --git a/src/nodes/input/pointer.rs b/src/nodes/input/pointer.rs index eac9718..6513439 100644 --- a/src/nodes/input/pointer.rs +++ b/src/nodes/input/pointer.rs @@ -1,14 +1,12 @@ +use super::{DistanceLink, InputSpecialization}; +use crate::nodes::fields::{ray_march, Field, Ray, RayMarchResult}; +use crate::nodes::spatial::Spatial; use glam::{vec3, Mat4}; use libstardustxr::schemas::input::InputDataRaw; use libstardustxr::schemas::input_pointer; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; -use crate::nodes::fields::{ray_march, Field, Ray, RayMarchResult}; -use crate::nodes::spatial::Spatial; - -use super::{DistanceLink, InputSpecialization}; - #[derive(Default)] pub struct Pointer { grab: AtomicBool, diff --git a/src/objects/input/mod.rs b/src/objects/input/mod.rs index b9be815..f6110dc 100644 --- a/src/objects/input/mod.rs +++ b/src/objects/input/mod.rs @@ -1 +1,2 @@ pub mod mouse_pointer; +pub mod sk_hand; diff --git a/src/objects/input/sk_hand.rs b/src/objects/input/sk_hand.rs new file mode 100644 index 0000000..1b5a553 --- /dev/null +++ b/src/objects/input/sk_hand.rs @@ -0,0 +1,72 @@ +use crate::nodes::{ + input::{InputMethod, InputType}, + spatial::Spatial, +}; +use glam::Mat4; +use libstardustxr::schemas::{common::JointT, input_hand::HandT}; +use std::sync::{Arc, Weak}; +use stereokit::{ + input::{Handed, Joint as SkJoint}, + StereoKit, +}; + +fn convert_joint(joint: SkJoint) -> JointT { + JointT { + position: joint.position.into(), + rotation: joint.orientation.into(), + radius: joint.radius, + } +} + +pub struct SkHand { + hand: Arc, + handed: Handed, +} +impl SkHand { + pub fn new(handed: Handed) -> Self { + let mut sk_hand = HandT::default(); + sk_hand.right = handed == Handed::Right; + SkHand { + hand: InputMethod::new( + Spatial::new(Weak::new(), None, Mat4::IDENTITY), + InputType::Hand(Box::new(sk_hand)), + ), + handed, + } + } + pub fn update(&mut self, sk: &StereoKit) { + if let InputType::Hand(hand) = &mut *self.hand.specialization.lock() { + let sk_hand = *sk.input_hand(self.handed); + // *self.hand.enabled.lock() = sk_hand.tracked_state.is_active(); + // if sk_hand.tracked_state.is_active() { + hand.thumb.tip = convert_joint(sk_hand.fingers[0][4]); + hand.thumb.distal = convert_joint(sk_hand.fingers[0][3]); + hand.thumb.proximal = convert_joint(sk_hand.fingers[0][2]); + hand.thumb.metacarpal = convert_joint(sk_hand.fingers[0][1]); + + for (finger, sk_finger) in [ + (&mut hand.index, sk_hand.fingers[1]), + (&mut hand.middle, sk_hand.fingers[2]), + (&mut hand.ring, sk_hand.fingers[3]), + (&mut hand.little, sk_hand.fingers[4]), + ] { + finger.tip = convert_joint(sk_finger[4]); + finger.distal = convert_joint(sk_finger[3]); + finger.intermediate = convert_joint(sk_finger[2]); + finger.proximal = convert_joint(sk_finger[1]); + finger.metacarpal = convert_joint(sk_finger[0]); + } + + hand.palm.position = sk_hand.palm.position.into(); + hand.palm.rotation = sk_hand.palm.orientation.into(); + hand.palm.radius = (sk_hand.fingers[2][0].radius + sk_hand.fingers[2][1].radius) * 0.5; + + hand.wrist.position = sk_hand.wrist.position.into(); + hand.wrist.rotation = sk_hand.wrist.orientation.into(); + hand.wrist.radius = (sk_hand.fingers[0][0].radius + sk_hand.fingers[4][0].radius) * 0.5; + + hand.elbow = None; + // } + } + } +}