refactor(objects): overhaul input

.gitignore

@@ -11,4 +11,5 @@
 *result*
 
 /libs/
-*.AppImage
+*.AppImage
+*.blend1

src/core/mod.rs

@@ -7,3 +7,4 @@ pub mod registry;
 pub mod resource;
 pub mod scenegraph;
 pub mod task;
+pub mod typed_datamap;

src/core/typed_datamap.rs

@@ -0,0 +1,54 @@
+#![allow(unused)]
+
+use std::ops::{Deref, DerefMut};
+
+use color_eyre::eyre::Result;
+use once_cell::sync::Lazy;
+use serde::{de::DeserializeOwned, Serialize};
+use stardust_xr::schemas::{
+	flat::Datamap,
+	flex::flexbuffers::{FlexbufferSerializer, Reader, ReaderError},
+};
+
+pub struct TypedDatamap<T: DeserializeOwned + Serialize>(T);
+impl<T: DeserializeOwned + Serialize> TypedDatamap<T> {
+	pub fn new(data: T) -> Self {
+		TypedDatamap(data)
+	}
+	pub fn from_flex(data: &[u8]) -> Result<Self> {
+		let root = Reader::get_root(data)?;
+		T::deserialize(root).map(Self::new).map_err(|e| e.into())
+	}
+	pub fn to_datamap(&mut self) -> Result<Datamap> {
+		let mut serializer = FlexbufferSerializer::default();
+		self.0.serialize(&mut serializer)?;
+		Datamap::new(serializer.take_buffer()).map_err(|e| e.into())
+	}
+	pub fn serialize(&mut self) -> Option<Vec<u8>> {
+		let mut serializer = FlexbufferSerializer::default();
+		self.0.serialize(&mut serializer).ok()?;
+		// check if this is actually a map
+		Reader::get_root(serializer.view()).ok()?.get_map().ok()?;
+		Some(serializer.take_buffer())
+	}
+}
+impl<T: DeserializeOwned + Serialize> Default for TypedDatamap<T>
+where
+	T: Default,
+{
+	fn default() -> Self {
+		Self(T::default())
+	}
+}
+impl<T: DeserializeOwned + Serialize> Deref for TypedDatamap<T> {
+	type Target = T;
+
+	fn deref(&self) -> &Self::Target {
+		&self.0
+	}
+}
+impl<T: DeserializeOwned + Serialize> DerefMut for TypedDatamap<T> {
+	fn deref_mut(&mut self) -> &mut Self::Target {
+		&mut self.0
+	}
+}

src/main.rs

@@ -144,7 +144,7 @@ fn main() {
 		}
 	}
 
-	let mouse_pointer = cli_args
+	let mut mouse_pointer = cli_args
 		.flatscreen
 		.then(MousePointer::new)
 		.transpose()
@@ -158,8 +158,8 @@ fn main() {
 		.flatten();
 	let mut controllers = (!cli_args.flatscreen && !cli_args.disable_controller)
 		.then(|| {
-			let left = SkController::new(Handed::Left).ok();
-			let right = SkController::new(Handed::Right).ok();
+			let left = SkController::new(&sk, Handed::Left).ok();
+			let right = SkController::new(&sk, Handed::Right).ok();
 			left.zip(right)
 		})
 		.flatten();
@@ -244,7 +244,7 @@ fn main() {
 				wayland.as_mut().unwrap().frame_event(sk);
 				destroy_queue::clear();
 
-				if let Some(mouse_pointer) = &mouse_pointer {
+				if let Some(mouse_pointer) = &mut mouse_pointer {
 					mouse_pointer.update(sk);
 				}
 				if let Some((left_hand, right_hand)) = &mut hands {

src/objects/input/mouse_pointer.rs

@@ -1,5 +1,5 @@
 use crate::{
-	core::client::INTERNAL_CLIENT,
+	core::{client::INTERNAL_CLIENT, typed_datamap::TypedDatamap},
 	nodes::{
 		data::{mask_matches, Mask, PulseSender, PULSE_RECEIVER_REGISTRY},
 		fields::Ray,
@@ -9,18 +9,25 @@ use crate::{
 	},
 };
 use color_eyre::eyre::Result;
-use glam::{vec3, Mat4, Vec3};
+use glam::{vec2, vec3, Mat4, Vec2, Vec3};
 use nanoid::nanoid;
-use serde::Serialize;
-use stardust_xr::schemas::{flat::Datamap, flex::flexbuffers};
+use serde::{Deserialize, Serialize};
+use stardust_xr::schemas::flex::flexbuffers;
 use std::{convert::TryFrom, sync::Arc};
 use stereokit::{ray_from_mouse, ButtonState, Key, StereoKitMultiThread};
 use tracing::instrument;
 
 const SK_KEYMAP: &str = include_str!("sk.kmp");
 
+#[derive(Default, Deserialize, Serialize)]
+struct MouseDatamap {
+	select: f32,
+	grab: f32,
+	scroll: Vec2,
+}
+
 #[derive(Debug, Clone, Serialize)]
-pub struct KeyboardEvent {
+struct KeyboardEvent {
 	pub keyboard: String,
 	pub keymap: Option<String>,
 	pub keys_up: Option<Vec<u32>>,
@@ -31,6 +38,7 @@ pub struct MousePointer {
 	node: Arc<Node>,
 	spatial: Arc<Spatial>,
 	pointer: Arc<InputMethod>,
+	datamap: TypedDatamap<MouseDatamap>,
 	keyboard_sender: Arc<PulseSender>,
 }
 impl MousePointer {
@@ -53,11 +61,12 @@ impl MousePointer {
 			node,
 			spatial,
 			pointer,
+			datamap: Default::default(),
 			keyboard_sender,
 		})
 	}
 	#[instrument(level = "debug", name = "Update Flatscreen Pointer Ray", skip_all)]
-	pub fn update(&self, sk: &impl StereoKitMultiThread) {
+	pub fn update(&mut self, sk: &impl StereoKitMultiThread) {
 		let mouse = sk.input_mouse();
 
 		let ray = ray_from_mouse(mouse.pos).unwrap();
@@ -71,30 +80,18 @@ impl MousePointer {
 		);
 		{
 			// Set pointer input datamap
-			let mut fbb = flexbuffers::Builder::default();
-			let mut map = fbb.start_map();
-			map.push(
-				"select",
-				if sk.input_key(Key::MouseLeft).contains(ButtonState::ACTIVE) {
-					1.0f32
-				} else {
-					0.0f32
-				},
-			);
-			map.push(
-				"grab",
-				if sk.input_key(Key::MouseRight).contains(ButtonState::ACTIVE) {
-					1.0f32
-				} else {
-					0.0f32
-				},
-			);
-			let mut scroll_vec = map.start_vector("scroll");
-			scroll_vec.push(0_f32);
-			scroll_vec.push(mouse.scroll_change / 120.0);
-			scroll_vec.end_vector();
-			map.end_map();
-			*self.pointer.datamap.lock() = Datamap::new(fbb.take_buffer()).ok();
+			self.datamap.select = if sk.input_key(Key::MouseLeft).contains(ButtonState::ACTIVE) {
+				1.0f32
+			} else {
+				0.0f32
+			};
+			self.datamap.grab = if sk.input_key(Key::MouseRight).contains(ButtonState::ACTIVE) {
+				1.0f32
+			} else {
+				0.0f32
+			};
+			self.datamap.scroll = vec2(0.0, mouse.scroll_change / 120.0);
+			*self.pointer.datamap.lock() = self.datamap.to_datamap().ok();
 		}
 		self.send_keyboard_input(sk);
 	}

src/objects/input/sk_controller.rs

@@ -1,5 +1,5 @@
 use crate::{
-	core::client::INTERNAL_CLIENT,
+	core::{client::INTERNAL_CLIENT, typed_datamap::TypedDatamap},
 	nodes::{
 		input::{tip::Tip, InputMethod, InputType},
 		spatial::Spatial,
@@ -7,51 +7,70 @@ use crate::{
 	},
 };
 use color_eyre::eyre::Result;
-use glam::Mat4;
+use glam::{Mat4, Vec2};
 use nanoid::nanoid;
-use stardust_xr::{
-	schemas::{flat::Datamap, flex::flexbuffers},
-	values::Transform,
-};
+use serde::{Deserialize, Serialize};
+use stardust_xr::values::Transform;
 use std::sync::Arc;
-use stereokit::{ButtonState, Handed, StereoKitMultiThread};
+use stereokit::{
+	ButtonState, Color128, Handed, Model, RenderLayer, StereoKitDraw, StereoKitMultiThread,
+};
 use tracing::instrument;
 
+#[derive(Default, Deserialize, Serialize)]
+struct ControllerDatamap {
+	select: f32,
+	grab: f32,
+	scroll: Vec2,
+}
+
 pub struct SkController {
 	_node: Arc<Node>,
 	input: Arc<InputMethod>,
+	model: Model,
 	handed: Handed,
+	datamap: TypedDatamap<ControllerDatamap>,
 }
 impl SkController {
-	pub fn new(handed: Handed) -> Result<Self> {
+	pub fn new(sk: &impl StereoKitMultiThread, handed: Handed) -> Result<Self> {
 		let _node = Node::create(&INTERNAL_CLIENT, "", &nanoid!(), false).add_to_scenegraph()?;
 		Spatial::add_to(&_node, None, Mat4::IDENTITY, false)?;
+		let model = sk.model_create_mem("cursor", include_bytes!("cursor.glb"), None)?;
 		let tip = InputType::Tip(Tip::default());
 		let input = InputMethod::add_to(&_node, tip, None)?;
 		Ok(SkController {
 			_node,
 			input,
 			handed,
+			model,
+			datamap: Default::default(),
 		})
 	}
 	#[instrument(level = "debug", name = "Update StereoKit Tip Input Method", skip_all)]
-	pub fn update(&mut self, sk: &impl StereoKitMultiThread) {
+	pub fn update(&mut self, sk: &impl StereoKitDraw) {
 		let controller = sk.input_controller(self.handed);
 		*self.input.enabled.lock() = controller.tracked.contains(ButtonState::ACTIVE);
 		if *self.input.enabled.lock() {
+			sk.model_draw(
+				&self.model,
+				Mat4::from_rotation_translation(
+					controller.aim.orientation,
+					controller.aim.position,
+				),
+				Color128::default(),
+				RenderLayer::all(),
+			);
 			self.input.spatial.set_local_transform_components(
 				None,
 				Transform::from_position_rotation(
-					controller.pose.position,
-					controller.pose.orientation,
+					controller.aim.position,
+					controller.aim.orientation,
 				),
 			);
 		}
-		let mut fbb = flexbuffers::Builder::default();
-		let mut map = fbb.start_map();
-		map.push("select", controller.trigger);
-		map.push("grab", controller.grip);
-		map.end_map();
-		*self.input.datamap.lock() = Datamap::new(fbb.take_buffer()).ok();
+		self.datamap.select = controller.trigger;
+		self.datamap.grab = controller.grip;
+		self.datamap.scroll = controller.stick;
+		*self.input.datamap.lock() = self.datamap.to_datamap().ok();
 	}
 }

src/objects/input/sk_hand.rs

@@ -1,5 +1,5 @@
 use crate::{
-	core::client::INTERNAL_CLIENT,
+	core::{client::INTERNAL_CLIENT, typed_datamap::TypedDatamap},
 	nodes::{
 		input::{hand::Hand, InputMethod, InputType},
 		spatial::Spatial,
@@ -9,10 +9,8 @@ use crate::{
 use color_eyre::eyre::Result;
 use glam::Mat4;
 use nanoid::nanoid;
-use stardust_xr::schemas::{
-	flat::{Datamap, Hand as FlatHand, Joint},
-	flex::flexbuffers,
-};
+use serde::{Deserialize, Serialize};
+use stardust_xr::schemas::flat::{Hand as FlatHand, Joint};
 use std::sync::Arc;
 use stereokit::{ButtonState, HandJoint, Handed, StereoKitMultiThread};
 use tracing::instrument;
@@ -25,10 +23,17 @@ fn convert_joint(joint: HandJoint) -> Joint {
 	}
 }
 
+#[derive(Default, Deserialize, Serialize)]
+struct HandDatamap {
+	pinch_strength: f32,
+	grab_strength: f32,
+}
+
 pub struct SkHand {
 	_node: Arc<Node>,
 	input: Arc<InputMethod>,
 	handed: Handed,
+	datamap: TypedDatamap<HandDatamap>,
 }
 impl SkHand {
 	pub fn new(handed: Handed) -> Result<Self> {
@@ -45,6 +50,7 @@ impl SkHand {
 			_node,
 			input,
 			handed,
+			datamap: Default::default(),
 		})
 	}
 	#[instrument(level = "debug", name = "Update Hand Input Method", skip_all)]
@@ -54,6 +60,7 @@ impl SkHand {
 			let controller = sk.input_controller(self.handed);
 			*self.input.enabled.lock() = controller.tracked.contains(ButtonState::INACTIVE)
 				&& sk_hand.tracked_state.contains(ButtonState::ACTIVE);
+			sk.input_hand_visible(self.handed, *self.input.enabled.lock());
 			if *self.input.enabled.lock() {
 				hand.base.thumb.tip = convert_joint(sk_hand.fingers[0][4]);
 				hand.base.thumb.distal = convert_joint(sk_hand.fingers[0][3]);
@@ -86,11 +93,8 @@ impl SkHand {
 				hand.base.elbow = None;
 			}
 		}
-		let mut fbb = flexbuffers::Builder::default();
-		let mut map = fbb.start_map();
-		map.push("grab_strength", sk_hand.grip_activation);
-		map.push("pinch_strength", sk_hand.pinch_activation);
-		map.end_map();
-		*self.input.datamap.lock() = Datamap::new(fbb.take_buffer()).ok();
+		self.datamap.pinch_strength = sk_hand.pinch_activation;
+		self.datamap.grab_strength = sk_hand.grip_activation;
+		*self.input.datamap.lock() = self.datamap.to_datamap().ok();
 	}
 }