From cbb54fd3d22dfb1407b83218c2b967ab32bb1c57 Mon Sep 17 00:00:00 2001 From: Schmarni Date: Tue, 17 Jun 2025 14:29:22 +0200 Subject: [PATCH] feat: working handtracking input methodsRUST_LOG=info,naga=warn dbus-run-session cargo run -- -e ~/build/stardust/env.sh -o 300! Signed-off-by: Schmarni --- Cargo.lock | 21 +-- src/main.rs | 135 +++++++++++--- src/objects/input/sk_hand.rs | 353 ++++++++++++++++++++++++++--------- src/objects/mod.rs | 38 ++-- src/objects/play_space.rs | 160 ++++++++++++++-- 5 files changed, 542 insertions(+), 165 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3b36cee..6caba97 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1179,7 +1179,7 @@ dependencies = [ [[package]] name = "bevy_mod_openxr" version = "0.3.0" -source = "git+https://github.com/Schmarni-Dev/bevy_openxr?branch=non_default_wait_frame_system#7a30bb2b523739501ba491956b96d9d6464ed8c5" +source = "git+https://github.com/Schmarni-Dev/bevy_openxr?branch=non_default_wait_frame_system#f0fd9a3086f1d898cf8d1a5338a3deae7ef66e2f" dependencies = [ "android_system_properties", "ash", @@ -1197,7 +1197,7 @@ dependencies = [ [[package]] name = "bevy_mod_xr" version = "0.3.0" -source = "git+https://github.com/Schmarni-Dev/bevy_openxr?branch=non_default_wait_frame_system#7a30bb2b523739501ba491956b96d9d6464ed8c5" +source = "git+https://github.com/Schmarni-Dev/bevy_openxr?branch=non_default_wait_frame_system#f0fd9a3086f1d898cf8d1a5338a3deae7ef66e2f" dependencies = [ "bevy", ] @@ -1696,7 +1696,7 @@ dependencies = [ "bitflags 2.9.1", "cexpr", "clang-sys", - "itertools 0.13.0", + "itertools 0.11.0", "log", "prettyplease", "proc-macro2", @@ -3550,15 +3550,6 @@ dependencies = [ "either", ] -[[package]] -name = "itertools" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.14.0" @@ -4259,7 +4250,7 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af1844ef2428cc3e1cb900be36181049ef3d3193c63e43026cfe202983b27a56" dependencies = [ - "proc-macro-crate 3.3.0", + "proc-macro-crate 1.3.1", "proc-macro2", "quote", "syn 2.0.101", @@ -4971,7 +4962,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.11.0", "proc-macro2", "quote", "syn 2.0.101", @@ -6925,7 +6916,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.48.0", ] [[package]] diff --git a/src/main.rs b/src/main.rs index ee7fb94..756c246 100644 --- a/src/main.rs +++ b/src/main.rs @@ -12,10 +12,11 @@ use crate::nodes::{audio, drawable, input}; use bevy::MinimalPlugins; use bevy::a11y::AccessibilityPlugin; -use bevy::app::{App, ScheduleRunnerPlugin, TerminalCtrlCHandlerPlugin}; +use bevy::app::{App, MainScheduleOrder, ScheduleRunnerPlugin, TerminalCtrlCHandlerPlugin}; use bevy::asset::{AssetMetaCheck, UnapprovedPathMode}; use bevy::audio::AudioPlugin; use bevy::core_pipeline::CorePipelinePlugin; +use bevy::ecs::schedule::{ExecutorKind, ScheduleLabel}; use bevy::gizmos::GizmoPlugin; use bevy::gltf::GltfPlugin; use bevy::input::{InputPlugin, InputSystem}; @@ -27,14 +28,21 @@ use bevy::scene::ScenePlugin; use bevy::text::FontLoader; use bevy::winit::{WakeUp, WinitPlugin}; use bevy_mod_meshtext::MeshTextPlugin; +use bevy_mod_openxr::action_binding::OxrActionBindingPlugin; +use bevy_mod_openxr::action_set_attaching::OxrActionAttachingPlugin; +use bevy_mod_openxr::action_set_syncing::OxrActionSyncingPlugin; use bevy_mod_openxr::add_xr_plugins; use bevy_mod_openxr::exts::OxrExtensions; +use bevy_mod_openxr::features::handtracking::HandTrackingPlugin; use bevy_mod_openxr::features::overlay::OxrOverlaySettings; -use bevy_mod_openxr::init::OxrInitPlugin; +use bevy_mod_openxr::features::passthrough::OxrPassthroughPlugin; +use bevy_mod_openxr::init::{OxrInitPlugin, should_run_frame_loop}; use bevy_mod_openxr::reference_space::OxrReferenceSpacePlugin; -use bevy_mod_openxr::resources::OxrSessionConfig; +use bevy_mod_openxr::render::{OxrRenderPlugin, OxrWaitFrameSystem}; +use bevy_mod_openxr::resources::{OxrFrameState, OxrFrameWaiter, OxrSessionConfig}; use bevy_mod_openxr::types::AppInfo; use bevy_mod_xr::hand_debug_gizmos::HandGizmosPlugin; +use bevy_mod_xr::session::{XrFirst, XrHandleEvents, session_running}; use clap::Parser; use core::client::{Client, tick_internal_client}; use core::task; @@ -42,6 +50,8 @@ use directories::ProjectDirs; use nodes::drawable::model::ModelNodePlugin; use nodes::spatial::SpatialNodePlugin; use objects::ServerObjects; +use objects::input::sk_hand::HandPlugin; +use objects::play_space::PlaySpacePlugin; use openxr::{EnvironmentBlendMode, ReferenceSpaceType}; use session::{launch_start, save_session}; use stardust_xr::schemas::dbus::object_registry::ObjectRegistry; @@ -239,6 +249,13 @@ async fn main() { static DEFAULT_SKYTEX: OnceLock = OnceLock::new(); static DEFAULT_SKYLIGHT: OnceLock = OnceLock::new(); +#[derive(ScheduleLabel, Hash, Debug, PartialEq, Eq, Clone, Copy)] +pub struct PreFrameWait; +#[derive(Resource, Deref)] +pub struct ObjectRegistryRes(ObjectRegistry); +#[derive(Resource, Deref)] +pub struct DbusConnection(Connection); + fn bevy_loop( ready_notifier: Arc, project_dirs: Option, @@ -247,6 +264,7 @@ fn bevy_loop( object_registry: ObjectRegistry, ) { let mut app = App::new(); + app.insert_resource(DbusConnection(dbus_connection)); app.add_plugins(AssetPlugin { meta_check: AssetMetaCheck::Never, unapproved_path_mode: UnapprovedPathMode::Allow, @@ -254,18 +272,18 @@ fn bevy_loop( }); let mut plugins = MinimalPlugins .build() - .disable::() + // .disable::() .add(TransformPlugin) .add(InputPlugin) /* .add(AccessibilityPlugin) */; // TODO: figure out headless - { - plugins = plugins.add(WindowPlugin::default()).add({ - let mut winit = WinitPlugin::::default(); - winit.run_on_any_thread = true; - winit - }); - } + // { + // plugins = plugins.add(WindowPlugin::default()).add({ + // let mut winit = WinitPlugin::::default(); + // winit.run_on_any_thread = true; + // winit + // }); + // } plugins = plugins .add(TerminalCtrlCHandlerPlugin) // bevy_mod_openxr will replace this, TODO: figure out how to mix this with @@ -288,8 +306,23 @@ fn bevy_loop( .add(AudioPlugin::default()) .add(GizmoPlugin) .add(AccessibilityPlugin); + let mut task_pool_plugin = TaskPoolPlugin::default(); + // make tokio work + let handle = tokio::runtime::Handle::current(); + let enter_runtime_context = Arc::new(move || { + // TODO: this might be a memory leak + std::mem::forget(handle.enter()); + }); + task_pool_plugin.task_pool_options.io.on_thread_spawn = Some(enter_runtime_context.clone()); + task_pool_plugin.task_pool_options.compute.on_thread_spawn = + Some(enter_runtime_context.clone()); + task_pool_plugin + .task_pool_options + .async_compute + .on_thread_spawn = Some(enter_runtime_context.clone()); + plugins = plugins.set(task_pool_plugin); app.add_plugins( - add_xr_plugins(plugins) + add_xr_plugins(plugins.add(WindowPlugin::default())) .set(OxrInitPlugin { app_info: AppInfo { name: "Stardust XR".into(), @@ -306,17 +339,21 @@ fn bevy_loop( }, ..default() }) + .set(OxrRenderPlugin { + default_wait_frame: false, + ..default() + }) .set(OxrReferenceSpacePlugin { default_primary_ref_space: ReferenceSpaceType::LOCAL, - }), + }) + // Disable a bunch of unneeded plugins + // this plugin uses the fb extention, blend mode still works + .disable::() + // we don't do any action stuff that needs to integrate with the ecosystem + .disable::() + .disable::() + .disable::(), ); - app.init_asset::().init_asset_loader::(); - if let Some(priority) = args.overlay_priority { - app.insert_resource(OxrOverlaySettings { - session_layer_placement: priority, - ..default() - }); - } app.add_plugins(( bevy_sk::hand::HandPlugin, bevy_sk::vr_materials::SkMaterialPlugin { @@ -325,7 +362,14 @@ fn bevy_loop( bevy_sk::skytext::SphericalHarmonicsPlugin, )); app.add_plugins(HandGizmosPlugin); - app.add_plugins(MeshTextPlugin); + // app.add_plugins(MeshTextPlugin); + // app.init_asset::().init_asset_loader::(); + if let Some(priority) = args.overlay_priority { + app.insert_resource(OxrOverlaySettings { + session_layer_placement: priority, + ..default() + }); + } app.insert_resource(OxrSessionConfig { blend_modes: Some(vec![ EnvironmentBlendMode::ALPHA_BLEND, @@ -334,26 +378,63 @@ fn bevy_loop( ]), ..default() }); + let mut pre_frame_wait = Schedule::new(PreFrameWait); + pre_frame_wait.set_executor_kind(ExecutorKind::MultiThreaded); + app.add_schedule(pre_frame_wait); app.insert_resource(ClearColor(Color::BLACK.with_alpha(0.0))); + app.insert_resource(ObjectRegistryRes(object_registry)); app.add_plugins((RemotePlugin::default(), RemoteHttpPlugin::default())); - app.add_plugins((SpatialNodePlugin, ModelNodePlugin)); + // the Stardust server plugins + app.add_plugins(( + SpatialNodePlugin, + ModelNodePlugin, + PlaySpacePlugin, + HandPlugin, + )); ready_notifier.notify_waiters(); - app.add_systems(PreUpdate, main_loop_system.after(InputSystem)); + app.add_systems( + XrFirst, + xr_step + .in_set(OxrWaitFrameSystem) + .in_set(XrHandleEvents::FrameLoop), + ); app.run(); } -fn main_loop_system(world: &mut World) { +fn xr_step(world: &mut World) { // camera::update(token); #[cfg(feature = "wayland")] wayland.frame_event(); destroy_queue::clear(); - // objects.update(&sk, token, &dbus_connection, &object_registry); - // input::process_input(); + // update things like the Xr input methods + world.run_schedule(PreFrameWait); + input::process_input(); let time = world.resource::().delta_secs_f64(); nodes::root::Root::send_frame_events(time); - // Wait + // we are targeting the frame after the wait + if let Some(mut state) = world.get_resource_mut::() { + state.predicted_display_time = openxr::Time::from_nanos( + state.predicted_display_time.as_nanos() + state.predicted_display_period.as_nanos(), + ); + } + + let should_wait = world + .run_system_cached(should_run_frame_loop) + .unwrap_or(false); + if should_wait { + world.resource_scope::(|world, mut waiter| { + let state = waiter + .wait() + .inspect_err(|err| error!("failed to wait OpenXR frame: {err}")) + .ok(); + + if let Some(state) = state { + world.insert_resource(OxrFrameState(state)); + } + }); + } tick_internal_client(); #[cfg(feature = "wayland")] diff --git a/src/objects/input/sk_hand.rs b/src/objects/input/sk_hand.rs index e6653fe..c55a2a5 100644 --- a/src/objects/input/sk_hand.rs +++ b/src/objects/input/sk_hand.rs @@ -1,15 +1,27 @@ use crate::core::client::INTERNAL_CLIENT; use crate::nodes::OwnedNode; use crate::nodes::fields::{Field, FieldTrait}; -use crate::nodes::input::{INPUT_HANDLER_REGISTRY, InputDataType, InputHandler}; +use crate::nodes::input::{Finger, INPUT_HANDLER_REGISTRY, InputDataType, InputHandler, Thumb}; use crate::nodes::{ Node, input::{Hand, InputMethod, Joint}, spatial::Spatial, }; use crate::objects::{ObjectHandle, SpatialRef, Tracked}; +use crate::{DbusConnection, ObjectRegistryRes, PreFrameWait}; +use bevy::prelude::Transform as BevyTransform; +use bevy::prelude::*; +use bevy_mod_openxr::helper_traits::{ToQuat, ToVec3}; +use bevy_mod_openxr::resources::OxrFrameState; +use bevy_mod_openxr::session::OxrSession; +use bevy_mod_xr::hands::{HandBone, HandSide, XrHandBoneEntities, XrHandBoneRadius}; +use bevy_mod_xr::session::{XrPreDestroySession, XrSessionCreated}; +use bevy_mod_xr::spaces::{XrPrimaryReferenceSpace, XrSpaceLocationFlags}; +use bevy_sk::hand::GRADIENT_TEXTURE_HANDLE; +use bevy_sk::vr_materials::PbrMaterial; use color_eyre::eyre::Result; use glam::{Mat4, Quat, Vec3}; +use openxr::{HandJointLocation, SpaceLocationFlags}; use serde::{Deserialize, Serialize}; use stardust_xr::values::Datamap; use std::sync::Arc; @@ -21,15 +33,146 @@ use zbus::Connection; use super::{CaptureManager, get_sorted_handlers}; -fn convert_joint(joint: HandJoint) -> Joint { +pub struct HandPlugin; +impl Plugin for HandPlugin { + fn build(&self, app: &mut App) { + app.add_systems(PreFrameWait, update_hands); + app.add_systems(XrSessionCreated, create_trackers); + app.add_systems(XrPreDestroySession, destroy_trackers); + app.add_systems(PostUpdate, update_hand_material); + app.add_systems(Startup, setup); + } +} +fn update_hands( + mut hands: ResMut, + session: Option>, + state: Option>, + ref_space: Option>, + mut materials: ResMut>, + mut joint_query: Query<( + &mut BevyTransform, + &mut XrSpaceLocationFlags, + &mut XrHandBoneRadius, + )>, + joints_query: Query<&XrHandBoneEntities>, +) { + let (Some(session), Some(state), Some(ref_space)) = (session, state, ref_space) else { + tokio::task::spawn({ + let left = hands.left.tracked.clone(); + let right = hands.right.tracked.clone(); + async move { + left.set_tracked(false); + right.set_tracked(false); + } + }); + return; + }; + let get_joints = |hand: &mut SkHand| -> Option { + let Some(tracker) = hand.tracker.as_ref() else { + hand.input.spatial.node().unwrap().set_enabled(false); + let handle = hand.tracked.clone(); + tokio::task::spawn(async move { + handle.set_tracked(false); + }); + return None; + }; + // this won't be correct with pipelined rendering + session + .locate_hand_joints(tracker, &ref_space, state.predicted_display_time) + .inspect_err(|err| error!("Error while locating hand joints")) + .ok() + .flatten() + }; + let joints_left = get_joints(&mut hands.left); + let joints_right = get_joints(&mut hands.right); + hands.left.update(joints_left.as_ref(), &mut materials); + hands.right.update(joints_right.as_ref(), &mut materials); +} + +fn pinch_between(joint_1: &Joint, joint_2: &Joint) -> f32 { + const PINCH_MAX: f32 = 0.11; + const PINCH_ACTIVACTION_DISTANCE: f32 = 0.01; + let combined_radius = joint_1.radius + joint_2.radius; + let pinch_dist = + Vec3::from(joint_1.position).distance(Vec3::from(joint_2.position)) - combined_radius; + (1.0 - ((pinch_dist - PINCH_ACTIVACTION_DISTANCE) / (PINCH_MAX - PINCH_ACTIVACTION_DISTANCE))) + .clamp(0.0, 1.0) +} + +fn create_trackers(session: Res, mut hands: ResMut) { + hands.left.tracker = session + .create_hand_tracker(openxr::HandEXT::LEFT) + .inspect_err(|err| error!("failed to create left hand tracker")) + .ok(); + hands.right.tracker = session + .create_hand_tracker(openxr::HandEXT::RIGHT) + .inspect_err(|err| error!("failed to create right hand tracker")) + .ok(); +} +fn destroy_trackers(mut hands: ResMut) { + hands.left.tracker.take(); + hands.right.tracker.take(); +} +#[derive(Component)] +struct CorrectHandMaterial; +fn update_hand_material( + query: Query< + (Entity, &HandSide), + ( + With, + With>, + Without, + ), + >, + mut cmds: Commands, + hands: Res, +) { + for (entity, side) in &query { + let handle = match side { + HandSide::Left => hands.left.material.clone(), + HandSide::Right => hands.right.material.clone(), + }; + cmds.entity(entity) + .insert(MeshMaterial3d(handle)) + .insert(CorrectHandMaterial); + } +} + +fn setup( + connection: Res, + mut cmds: Commands, + mut materials: ResMut>, +) { + tokio::task::spawn({ + let connection = connection.clone(); + async move { + connection + .request_name("org.stardustxr.Hands") + .await + .unwrap(); + } + }); + cmds.insert_resource(Hands { + left: SkHand::new(&connection, HandSide::Left, &mut materials).unwrap(), + right: SkHand::new(&connection, HandSide::Right, &mut materials).unwrap(), + }); +} + +fn convert_joint(joint: HandJointLocation) -> Joint { Joint { - position: Vec3::from(joint.position).into(), - rotation: Quat::from(joint.orientation).into(), + position: joint.pose.position.to_vec3().into(), + rotation: joint.pose.orientation.to_quat().into(), radius: joint.radius, distance: 0.0, } } +#[derive(Resource)] +struct Hands { + left: SkHand, + right: SkHand, +} + #[derive(Default, Deserialize, Serialize)] struct HandDatamap { pinch_strength: f32, @@ -40,117 +183,158 @@ pub struct SkHand { _node: OwnedNode, palm_spatial: Arc, palm_object: ObjectHandle, - handed: Handed, + side: HandSide, input: Arc, capture_manager: CaptureManager, datamap: HandDatamap, tracked: ObjectHandle, + tracker: Option, + captured: bool, + material: Handle, } impl SkHand { - pub fn new(connection: &Connection, handed: Handed) -> Result { + pub fn new( + connection: &Connection, + side: HandSide, + materials: &mut Assets, + ) -> Result { let (palm_spatial, palm_object) = SpatialRef::create( connection, &("/org/stardustxr/Hand/".to_string() - + match handed { - Handed::Left => "left", - _ => "right", + + match side { + HandSide::Left => "left", + HandSide::Right => "right", } + "/palm"), ); let tracked = Tracked::new( connection, &("/org/stardustxr/Hand/".to_string() - + match handed { - Handed::Left => "left", - _ => "right", + + match side { + HandSide::Left => "left", + HandSide::Right => "right", }), ); - let _node = Node::generate(&INTERNAL_CLIENT, false).add_to_scenegraph_owned()?; - Spatial::add_to(&_node.0, None, Mat4::IDENTITY, false); + let node = Node::generate(&INTERNAL_CLIENT, false).add_to_scenegraph_owned()?; + Spatial::add_to(&node.0, None, Mat4::IDENTITY, false); let hand = InputDataType::Hand(Hand { - right: handed == Handed::Right, + right: matches!(side, HandSide::Right), ..Default::default() }); let datamap = Datamap::from_typed(HandDatamap::default())?; - let input = InputMethod::add_to(&_node.0, hand, datamap)?; - Input::hand_visible(handed, true); + let input = InputMethod::add_to(&node.0, hand, datamap)?; + let material = materials.add(PbrMaterial { + color: Srgba::new(1.0, 1.0, 1.0, 1.0).into(), + alpha_mode: AlphaMode::Blend, + use_stereokit_uvs: false, + diffuse_texture: Some(GRADIENT_TEXTURE_HANDLE), + roughness: 1.0, + ..default() + }); Ok(SkHand { - _node, + _node: node, palm_spatial, palm_object, - handed, + side, input, tracked, capture_manager: CaptureManager::default(), datamap: Default::default(), + tracker: None, + material, + captured: false, }) } - pub fn update(&mut self, sk: &Sk, token: &MainThreadToken, material: &mut Material) { - let sk_hand = Input::hand(self.handed); - let real_hand = Input::hand_source(self.handed) as u32 == HandSource::Articulated as u32; - if let InputDataType::Hand(hand) = &mut *self.input.data.lock() { - let input_node = self.input.spatial.node().unwrap(); - input_node.set_enabled( - (real_hand || sk.get_active_display_mode() == DisplayMode::Flatscreen) - && sk_hand.tracked.is_active(), - ); - let enabled = input_node.enabled(); - tokio::spawn({ - // this is suboptimal since it probably allocates a fresh string every frame - let handle = self.tracked.clone(); - async move { - handle.set_tracked(enabled).await; - } + fn update( + &mut self, + joints: Option<&openxr::HandJointLocations>, + materials: &mut ResMut>, + ) { + // TODO: use the hand data source ext + let real_hand = true; + let input_node = self.input.spatial.node().unwrap(); + let is_tracked = real_hand + && joints.is_some_and(|v| { + v.iter().all(|v| { + v.location_flags.contains( + SpaceLocationFlags::POSITION_VALID + | SpaceLocationFlags::POSITION_TRACKED + | SpaceLocationFlags::ORIENTATION_VALID + | SpaceLocationFlags::ORIENTATION_TRACKED, + ) + }) }); - if enabled { - hand.thumb.tip = convert_joint(sk_hand.fingers[0][4]); - hand.thumb.distal = convert_joint(sk_hand.fingers[0][3]); - hand.thumb.proximal = convert_joint(sk_hand.fingers[0][2]); - hand.thumb.metacarpal = convert_joint(sk_hand.fingers[0][1]); - - for (finger, mut sk_finger) in [ - (&mut hand.index, sk_hand.fingers[1]), - (&mut hand.middle, sk_hand.fingers[2]), - (&mut hand.ring, sk_hand.fingers[3]), - (&mut hand.little, sk_hand.fingers[4]), - ] { - sk_finger[4].radius = 0.0; - finger.tip = convert_joint(sk_finger[4]); - finger.distal = convert_joint(sk_finger[3]); - finger.intermediate = convert_joint(sk_finger[2]); - finger.proximal = convert_joint(sk_finger[1]); - finger.metacarpal = convert_joint(sk_finger[0]); - } - - hand.palm.position = Vec3::from(sk_hand.palm.position).into(); - hand.palm.rotation = Quat::from(sk_hand.palm.orientation).into(); - hand.palm.radius = - (sk_hand.fingers[2][0].radius + sk_hand.fingers[2][1].radius) * 0.5; - - self.palm_spatial - .set_local_transform(Mat4::from_rotation_translation( - hand.palm.rotation.into(), - hand.palm.position.into(), - )); - - hand.wrist.position = Vec3::from(sk_hand.wrist.position).into(); - hand.wrist.rotation = Quat::from(sk_hand.wrist.orientation).into(); - hand.wrist.radius = - (sk_hand.fingers[0][0].radius + sk_hand.fingers[4][0].radius) * 0.5; - - hand.elbow = None; - - let hand_color = if self.capture_manager.capture.upgrade().is_none() { - Color128::new_rgb(1.0, 1.0, 1.0) - } else { - Color128::new_rgb(0.0, 1.0, 0.75) - }; - material.color_tint(hand_color); + input_node.set_enabled(is_tracked); + tokio::task::spawn({ + let handle = self.tracked.clone(); + async move { + handle.set_tracked(is_tracked); } + }); + if is_tracked { + // cannot ever crash, is_tracked is only true of joints is some + let joints = joints.unwrap(); + let new_hand = Hand { + right: matches!(self.side, HandSide::Right), + thumb: Thumb { + tip: convert_joint(joints[HandBone::ThumbTip as usize]), + distal: convert_joint(joints[HandBone::ThumbDistal as usize]), + proximal: convert_joint(joints[HandBone::ThumbProximal as usize]), + metacarpal: convert_joint(joints[HandBone::ThumbMetacarpal as usize]), + }, + index: Finger { + tip: convert_joint(joints[HandBone::IndexTip as usize]), + distal: convert_joint(joints[HandBone::IndexDistal as usize]), + intermediate: convert_joint(joints[HandBone::IndexIntermediate as usize]), + proximal: convert_joint(joints[HandBone::IndexProximal as usize]), + metacarpal: convert_joint(joints[HandBone::IndexMetacarpal as usize]), + }, + middle: Finger { + tip: convert_joint(joints[HandBone::MiddleTip as usize]), + distal: convert_joint(joints[HandBone::MiddleDistal as usize]), + intermediate: convert_joint(joints[HandBone::MiddleIntermediate as usize]), + proximal: convert_joint(joints[HandBone::MiddleProximal as usize]), + metacarpal: convert_joint(joints[HandBone::MiddleMetacarpal as usize]), + }, + ring: Finger { + tip: convert_joint(joints[HandBone::RingTip as usize]), + distal: convert_joint(joints[HandBone::RingDistal as usize]), + intermediate: convert_joint(joints[HandBone::RingIntermediate as usize]), + proximal: convert_joint(joints[HandBone::RingProximal as usize]), + metacarpal: convert_joint(joints[HandBone::RingMetacarpal as usize]), + }, + little: Finger { + tip: convert_joint(joints[HandBone::LittleTip as usize]), + distal: convert_joint(joints[HandBone::LittleDistal as usize]), + intermediate: convert_joint(joints[HandBone::LittleIntermediate as usize]), + proximal: convert_joint(joints[HandBone::LittleProximal as usize]), + metacarpal: convert_joint(joints[HandBone::LittleMetacarpal as usize]), + }, + palm: convert_joint(joints[HandBone::Palm as usize]), + wrist: convert_joint(joints[HandBone::Wrist as usize]), + elbow: None, + }; + self.palm_spatial + .set_local_transform(Mat4::from_rotation_translation( + new_hand.palm.rotation.into(), + new_hand.palm.position.into(), + )); + + self.datamap.pinch_strength = pinch_between(&new_hand.thumb.tip, &new_hand.index.tip); + // this is how stereokit calculates grab + self.datamap.grab_strength = + pinch_between(&new_hand.ring.tip, &new_hand.ring.metacarpal); + + *self.input.data.lock() = InputDataType::Hand(new_hand); + *self.input.datamap.lock() = Datamap::from_typed(&self.datamap).unwrap(); + let captured = self.capture_manager.capture.upgrade().is_some(); + if captured && !self.captured { + materials.get_mut(&self.material).unwrap().color = Srgba::rgb(0., 1., 0.75).into(); + } else if self.captured && !captured { + materials.get_mut(&self.material).unwrap().color = Srgba::rgb(1., 1.0, 1.0).into(); + } + self.captured = captured; } - self.datamap.pinch_strength = sk_hand.pinch_activation; - self.datamap.grab_strength = sk_hand.grip_activation; - *self.input.datamap.lock() = Datamap::from_typed(&self.datamap).unwrap(); let distance_calculator = |space: &Arc, data: &InputDataType, field: &Field| { let InputDataType::Hand(hand) = data else { @@ -183,11 +367,6 @@ impl SkHand { .set_handler_order(sorted_handlers.iter().map(|(handler, _)| handler)); } } -impl Drop for SkHand { - fn drop(&mut self) { - Input::hand_visible(self.handed, false); - } -} fn joint_to_line_point(joint: &Joint, color: Color128) -> LinePoint { LinePoint { diff --git a/src/objects/mod.rs b/src/objects/mod.rs index b848b8e..aae5d36 100644 --- a/src/objects/mod.rs +++ b/src/objects/mod.rs @@ -13,6 +13,7 @@ use input::{ eye_pointer::EyePointer, mouse_pointer::MousePointer, sk_controller::SkController, sk_hand::SkHand, }; +use parking_lot::RwLock; use play_space::PlaySpaceBounds; use stardust_xr::schemas::dbus::object_registry::ObjectRegistry; use std::{ @@ -34,16 +35,16 @@ enum Inputs { XR { controller_left: SkController, controller_right: SkController, - hand_left: SkHand, - hand_right: SkHand, + // hand_left: SkHand, + // hand_right: SkHand, eye_pointer: Option, }, MousePointer(MousePointer), // Controllers((SkController, SkController)), - Hands { - left: SkHand, - right: SkHand, - }, + // Hands { + // left: SkHand, + // right: SkHand, + // }, } pub struct ServerObjects { @@ -69,7 +70,8 @@ impl ServerObjects { if play_space.is_some() { let dbus_connection = connection.clone(); tokio::task::spawn(async move { - PlaySpaceBounds::create(&dbus_connection).await; + let play_space_data = Arc::new(RwLock::default()); + PlaySpaceBounds::create(&dbus_connection, play_space_data).await; dbus_connection .request_name("org.stardustxr.PlaySpace") .await @@ -95,8 +97,8 @@ impl ServerObjects { Inputs::XR { controller_left: SkController::new(&connection, Handed::Left).unwrap(), controller_right: SkController::new(&connection, Handed::Right).unwrap(), - hand_left: SkHand::new(&connection, Handed::Left).unwrap(), - hand_right: SkHand::new(&connection, Handed::Right).unwrap(), + // hand_left: SkHand::new(&connection, Handed::Left).unwrap(), + // hand_right: SkHand::new(&connection, Handed::Right).unwrap(), eye_pointer: Device::has_eye_gaze() .then(EyePointer::new) .transpose() @@ -166,8 +168,6 @@ impl ServerObjects { Inputs::XR { controller_left, controller_right, - hand_left, - hand_right, eye_pointer, } => { if !self.disable_controllers { @@ -176,10 +176,10 @@ impl ServerObjects { } Input::hand_visible(Handed::Left, !self.disable_hands); Input::hand_visible(Handed::Right, !self.disable_hands); - if !self.disable_hands { - hand_left.update(sk, token, &mut self.hand_materials[0]); - hand_right.update(sk, token, &mut self.hand_materials[1]); - } + // if !self.disable_hands { + // hand_left.update(sk, token, &mut self.hand_materials[0]); + // hand_right.update(sk, token, &mut self.hand_materials[1]); + // } if let Some(eye_pointer) = eye_pointer { eye_pointer.update(); } @@ -191,10 +191,10 @@ impl ServerObjects { // left.update(token); // right.update(token); // } - Inputs::Hands { left, right } => { - left.update(sk, token, &mut self.hand_materials[0]); - right.update(sk, token, &mut self.hand_materials[1]); - } + // Inputs::Hands { left, right } => { + // left.update(sk, token, &mut self.hand_materials[0]); + // right.update(sk, token, &mut self.hand_materials[1]); + // } } } } diff --git a/src/objects/play_space.rs b/src/objects/play_space.rs index c392c9b..4639808 100644 --- a/src/objects/play_space.rs +++ b/src/objects/play_space.rs @@ -1,12 +1,151 @@ +use std::sync::Arc; + +use bevy::prelude::*; +use bevy_mod_openxr::{ + helper_traits::{ToQuat, ToVec3}, + resources::OxrFrameState, + session::OxrSession, +}; +use bevy_mod_xr::{ + session::{XrPreDestroySession, XrSessionCreated}, + spaces::{XrPrimaryReferenceSpace, XrReferenceSpace, XrSpace}, +}; +use openxr::SpaceLocationFlags; +use parking_lot::RwLock; use stereokit_rust::system::World; use zbus::{Connection, ObjectServer, interface}; -pub struct PlaySpaceBounds; +use crate::{DbusConnection, PreFrameWait, nodes::spatial::Spatial}; + +use super::{ObjectHandle, SpatialRef, Tracked}; + +pub struct PlaySpacePlugin; +impl Plugin for PlaySpacePlugin { + fn build(&self, app: &mut App) { + app.add_systems(XrPreDestroySession, destroy_stage_space); + app.add_systems(XrSessionCreated, create_stage_space); + app.add_systems(PreFrameWait, update); + app.add_systems(Startup, setup); + } +} + +fn setup(connection: Res, mut cmds: Commands) { + let (spatial, spatial_handle) = SpatialRef::create(&connection, "/org/stardustxr/PlaySpace"); + // the OpenXR session might not exist quite yet + let tracked = Tracked::new(&connection, "/org/stardustxr/PlaySpace"); + let dbus_connection = connection.clone(); + let play_space_data = Arc::new(RwLock::default()); + tokio::task::spawn({ + let data = play_space_data.clone(); + async move { + PlaySpaceBounds::create(&dbus_connection, data).await; + dbus_connection + .request_name("org.stardustxr.PlaySpace") + .await + .unwrap(); + } + }); + cmds.insert_resource(PlaySpace { + spatial, + _spatial_handle: spatial_handle, + tracked_handle: tracked, + bounds: play_space_data, + }); +} + +#[derive(Resource)] +struct StageSpace(XrSpace); +fn create_stage_space(session: Res, mut cmds: Commands) { + let space = session + .create_reference_space(openxr::ReferenceSpaceType::STAGE, Transform::IDENTITY) + .inspect_err(|err| error!("failed to create Stage XrSpace")) + .ok(); + if let Some(space) = space { + cmds.insert_resource(StageSpace(space.0)); + } +} +fn destroy_stage_space(session: Res, mut cmds: Commands, stage: Res) { + session.destroy_space(stage.0); + cmds.remove_resource::(); +} + +/// TODO: impl this +fn update( + session: Option>, + stage: Option>, + ref_space: Option>, + play_space: Res, + state: Option>, +) { + let (Some(session), Some(stage), Some(ref_space), Some(state)) = + (session, stage, ref_space, state) + else { + play_space.bounds.write().drain(..); + tokio::task::spawn({ + let handle = play_space.tracked_handle.clone(); + async move { + handle.set_tracked(false); + } + }); + return; + }; + // this won't be correct with pipelined rendering + let location = session + .locate_space(&stage.0, &ref_space, state.predicted_display_time) + .inspect_err(|err| error!("Error while Locating OpenXR Stage Space {err}")); + if let Ok(location) = location { + let is_tracked = location.location_flags.contains( + SpaceLocationFlags::POSITION_VALID + | SpaceLocationFlags::POSITION_TRACKED + | SpaceLocationFlags::ORIENTATION_VALID + | SpaceLocationFlags::ORIENTATION_TRACKED, + ); + tokio::task::spawn({ + let handle = play_space.tracked_handle.clone(); + async move { + handle.set_tracked(is_tracked); + } + }); + if is_tracked { + play_space + .spatial + .set_local_transform(Mat4::from_rotation_translation( + location.pose.orientation.to_quat(), + location.pose.position.to_vec3(), + )); + } + } + // session.reference_space_bounds_rect(openxr::ReferenceSpaceType::STAGE); + + // if (World::has_bounds() + // && World::get_bounds_size().x != 0.0 + // && World::get_bounds_size().y != 0.0) + // { + // let bounds = World::get_bounds_size(); + // vec![ + // ((bounds.x).into(), (bounds.y).into()), + // ((bounds.x).into(), (-bounds.y).into()), + // ((-bounds.x).into(), (-bounds.y).into()), + // ((-bounds.x).into(), (bounds.y).into()), + // ] + // } else { + // vec![] + // } +} + +#[derive(Resource)] +pub struct PlaySpace { + spatial: Arc, + _spatial_handle: ObjectHandle, + tracked_handle: ObjectHandle, + bounds: Arc>>, +} +pub struct PlaySpaceBounds(Arc>>); impl PlaySpaceBounds { - pub async fn create(connection: &Connection) { + pub async fn create(connection: &Connection, data: Arc>>) { connection .object_server() - .at("/org/stardustxr/PlaySpace", Self) + .at("/org/stardustxr/PlaySpace", Self(data)) .await .unwrap(); } @@ -15,19 +154,6 @@ impl PlaySpaceBounds { impl PlaySpaceBounds { #[zbus(property)] fn bounds(&self) -> Vec<(f64, f64)> { - if (World::has_bounds() - && World::get_bounds_size().x != 0.0 - && World::get_bounds_size().y != 0.0) - { - let bounds = World::get_bounds_size(); - vec![ - ((bounds.x).into(), (bounds.y).into()), - ((bounds.x).into(), (-bounds.y).into()), - ((-bounds.x).into(), (-bounds.y).into()), - ((-bounds.x).into(), (bounds.y).into()), - ] - } else { - vec![] - } + self.0.read().clone() } }