StardustXR/server
1
0
Fork 0
Code Issues 11 Pull Requests 3 Actions Packages Projects Releases 5 Wiki Activity

feat: working handtracking input methodsRUST_LOG=info,naga=warn dbus-run-session cargo run -- -e ~/build/stardust/env.sh -o 300!

Browse Source 
Signed-off-by: Schmarni <marnistromer@gmail.com>
This commit is contained in:
Schmarni
2025-06-17 14:29:22 +02:00
committed by Nova King
parent aeec63c070
commit cbb54fd3d2
5 changed files with 542 additions and 165 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
Cargo.lock generated
View File

@@ -1179,7 +1179,7 @@ dependencies = [
[[package]]
name = "bevy_mod_openxr"
version = "0.3.0"
source = "git+https://github.com/Schmarni-Dev/bevy_openxr?branch=non_default_wait_frame_system#7a30bb2b523739501ba491956b96d9d6464ed8c5"
source = "git+https://github.com/Schmarni-Dev/bevy_openxr?branch=non_default_wait_frame_system#f0fd9a3086f1d898cf8d1a5338a3deae7ef66e2f"
dependencies = [
"android_system_properties",
"ash",
@@ -1197,7 +1197,7 @@ dependencies = [
[[package]]
name = "bevy_mod_xr"
version = "0.3.0"
source = "git+https://github.com/Schmarni-Dev/bevy_openxr?branch=non_default_wait_frame_system#7a30bb2b523739501ba491956b96d9d6464ed8c5"
source = "git+https://github.com/Schmarni-Dev/bevy_openxr?branch=non_default_wait_frame_system#f0fd9a3086f1d898cf8d1a5338a3deae7ef66e2f"
dependencies = [
"bevy",
]
@@ -1696,7 +1696,7 @@ dependencies = [
"bitflags 2.9.1",
"cexpr",
"clang-sys",
"itertools 0.13.0",
"itertools 0.11.0",
"log",
"prettyplease",
"proc-macro2",
@@ -3550,15 +3550,6 @@ dependencies = [
"either",
]
[[package]]
name = "itertools"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
dependencies = [
"either",
]
[[package]]
name = "itertools"
version = "0.14.0"
@@ -4259,7 +4250,7 @@ version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af1844ef2428cc3e1cb900be36181049ef3d3193c63e43026cfe202983b27a56"
dependencies = [
"proc-macro-crate 3.3.0",
"proc-macro-crate 1.3.1",
"proc-macro2",
"quote",
"syn 2.0.101",
@@ -4971,7 +4962,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d"
dependencies = [
"anyhow",
"itertools 0.14.0",
"itertools 0.11.0",
"proc-macro2",
"quote",
"syn 2.0.101",
@@ -6925,7 +6916,7 @@ version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb"
dependencies = [
"windows-sys 0.59.0",
"windows-sys 0.48.0",
]
[[package]]

135
src/main.rs
View File

@@ -12,10 +12,11 @@ use crate::nodes::{audio, drawable, input};
use bevy::MinimalPlugins;
use bevy::a11y::AccessibilityPlugin;
use bevy::app::{App, ScheduleRunnerPlugin, TerminalCtrlCHandlerPlugin};
use bevy::app::{App, MainScheduleOrder, ScheduleRunnerPlugin, TerminalCtrlCHandlerPlugin};
use bevy::asset::{AssetMetaCheck, UnapprovedPathMode};
use bevy::audio::AudioPlugin;
use bevy::core_pipeline::CorePipelinePlugin;
use bevy::ecs::schedule::{ExecutorKind, ScheduleLabel};
use bevy::gizmos::GizmoPlugin;
use bevy::gltf::GltfPlugin;
use bevy::input::{InputPlugin, InputSystem};
@@ -27,14 +28,21 @@ use bevy::scene::ScenePlugin;
use bevy::text::FontLoader;
use bevy::winit::{WakeUp, WinitPlugin};
use bevy_mod_meshtext::MeshTextPlugin;
use bevy_mod_openxr::action_binding::OxrActionBindingPlugin;
use bevy_mod_openxr::action_set_attaching::OxrActionAttachingPlugin;
use bevy_mod_openxr::action_set_syncing::OxrActionSyncingPlugin;
use bevy_mod_openxr::add_xr_plugins;
use bevy_mod_openxr::exts::OxrExtensions;
use bevy_mod_openxr::features::handtracking::HandTrackingPlugin;
use bevy_mod_openxr::features::overlay::OxrOverlaySettings;
use bevy_mod_openxr::init::OxrInitPlugin;
use bevy_mod_openxr::features::passthrough::OxrPassthroughPlugin;
use bevy_mod_openxr::init::{OxrInitPlugin, should_run_frame_loop};
use bevy_mod_openxr::reference_space::OxrReferenceSpacePlugin;
use bevy_mod_openxr::resources::OxrSessionConfig;
use bevy_mod_openxr::render::{OxrRenderPlugin, OxrWaitFrameSystem};
use bevy_mod_openxr::resources::{OxrFrameState, OxrFrameWaiter, OxrSessionConfig};
use bevy_mod_openxr::types::AppInfo;
use bevy_mod_xr::hand_debug_gizmos::HandGizmosPlugin;
use bevy_mod_xr::session::{XrFirst, XrHandleEvents, session_running};
use clap::Parser;
use core::client::{Client, tick_internal_client};
use core::task;
@@ -42,6 +50,8 @@ use directories::ProjectDirs;
use nodes::drawable::model::ModelNodePlugin;
use nodes::spatial::SpatialNodePlugin;
use objects::ServerObjects;
use objects::input::sk_hand::HandPlugin;
use objects::play_space::PlaySpacePlugin;
use openxr::{EnvironmentBlendMode, ReferenceSpaceType};
use session::{launch_start, save_session};
use stardust_xr::schemas::dbus::object_registry::ObjectRegistry;
@@ -239,6 +249,13 @@ async fn main() {
static DEFAULT_SKYTEX: OnceLock<Tex> = OnceLock::new();
static DEFAULT_SKYLIGHT: OnceLock<SphericalHarmonics> = OnceLock::new();
#[derive(ScheduleLabel, Hash, Debug, PartialEq, Eq, Clone, Copy)]
pub struct PreFrameWait;
#[derive(Resource, Deref)]
pub struct ObjectRegistryRes(ObjectRegistry);
#[derive(Resource, Deref)]
pub struct DbusConnection(Connection);
fn bevy_loop(
ready_notifier: Arc<Notify>,
project_dirs: Option<ProjectDirs>,
@@ -247,6 +264,7 @@ fn bevy_loop(
object_registry: ObjectRegistry,
) {
let mut app = App::new();
app.insert_resource(DbusConnection(dbus_connection));
app.add_plugins(AssetPlugin {
meta_check: AssetMetaCheck::Never,
unapproved_path_mode: UnapprovedPathMode::Allow,
@@ -254,18 +272,18 @@ fn bevy_loop(
});
let mut plugins = MinimalPlugins
.build()
.disable::<ScheduleRunnerPlugin>()
// .disable::<ScheduleRunnerPlugin>()
.add(TransformPlugin)
.add(InputPlugin)
/* .add(AccessibilityPlugin) */;
// TODO: figure out headless
{
plugins = plugins.add(WindowPlugin::default()).add({
let mut winit = WinitPlugin::<WakeUp>::default();
winit.run_on_any_thread = true;
winit
});
}
// {
// plugins = plugins.add(WindowPlugin::default()).add({
// let mut winit = WinitPlugin::<WakeUp>::default();
// winit.run_on_any_thread = true;
// winit
// });
// }
plugins = plugins
.add(TerminalCtrlCHandlerPlugin)
// bevy_mod_openxr will replace this, TODO: figure out how to mix this with
@@ -288,8 +306,23 @@ fn bevy_loop(
.add(AudioPlugin::default())
.add(GizmoPlugin)
.add(AccessibilityPlugin);
let mut task_pool_plugin = TaskPoolPlugin::default();
// make tokio work
let handle = tokio::runtime::Handle::current();
let enter_runtime_context = Arc::new(move || {
// TODO: this might be a memory leak
std::mem::forget(handle.enter());
});
task_pool_plugin.task_pool_options.io.on_thread_spawn = Some(enter_runtime_context.clone());
task_pool_plugin.task_pool_options.compute.on_thread_spawn =
Some(enter_runtime_context.clone());
task_pool_plugin
.task_pool_options
.async_compute
.on_thread_spawn = Some(enter_runtime_context.clone());
plugins = plugins.set(task_pool_plugin);
app.add_plugins(
add_xr_plugins(plugins)
add_xr_plugins(plugins.add(WindowPlugin::default()))
.set(OxrInitPlugin {
app_info: AppInfo {
name: "Stardust XR".into(),
@@ -306,17 +339,21 @@ fn bevy_loop(
},
..default()
})
.set(OxrRenderPlugin {
default_wait_frame: false,
..default()
})
.set(OxrReferenceSpacePlugin {
default_primary_ref_space: ReferenceSpaceType::LOCAL,
}),
})
// Disable a bunch of unneeded plugins
// this plugin uses the fb extention, blend mode still works
.disable::<OxrPassthroughPlugin>()
// we don't do any action stuff that needs to integrate with the ecosystem
.disable::<OxrActionAttachingPlugin>()
.disable::<OxrActionSyncingPlugin>()
.disable::<OxrActionBindingPlugin>(),
);
app.init_asset::<Font>().init_asset_loader::<FontLoader>();
if let Some(priority) = args.overlay_priority {
app.insert_resource(OxrOverlaySettings {
session_layer_placement: priority,
..default()
});
}
app.add_plugins((
bevy_sk::hand::HandPlugin,
bevy_sk::vr_materials::SkMaterialPlugin {
@@ -325,7 +362,14 @@ fn bevy_loop(
bevy_sk::skytext::SphericalHarmonicsPlugin,
));
app.add_plugins(HandGizmosPlugin);
app.add_plugins(MeshTextPlugin);
// app.add_plugins(MeshTextPlugin);
// app.init_asset::<Font>().init_asset_loader::<FontLoader>();
if let Some(priority) = args.overlay_priority {
app.insert_resource(OxrOverlaySettings {
session_layer_placement: priority,
..default()
});
}
app.insert_resource(OxrSessionConfig {
blend_modes: Some(vec![
EnvironmentBlendMode::ALPHA_BLEND,
@@ -334,26 +378,63 @@ fn bevy_loop(
]),
..default()
});
let mut pre_frame_wait = Schedule::new(PreFrameWait);
pre_frame_wait.set_executor_kind(ExecutorKind::MultiThreaded);
app.add_schedule(pre_frame_wait);
app.insert_resource(ClearColor(Color::BLACK.with_alpha(0.0)));
app.insert_resource(ObjectRegistryRes(object_registry));
app.add_plugins((RemotePlugin::default(), RemoteHttpPlugin::default()));
app.add_plugins((SpatialNodePlugin, ModelNodePlugin));
// the Stardust server plugins
app.add_plugins((
SpatialNodePlugin,
ModelNodePlugin,
PlaySpacePlugin,
HandPlugin,
));
ready_notifier.notify_waiters();
app.add_systems(PreUpdate, main_loop_system.after(InputSystem));
app.add_systems(
XrFirst,
xr_step
.in_set(OxrWaitFrameSystem)
.in_set(XrHandleEvents::FrameLoop),
);
app.run();
}
fn main_loop_system(world: &mut World) {
fn xr_step(world: &mut World) {
// camera::update(token);
#[cfg(feature = "wayland")]
wayland.frame_event();
destroy_queue::clear();
// objects.update(&sk, token, &dbus_connection, &object_registry);
// input::process_input();
// update things like the Xr input methods
world.run_schedule(PreFrameWait);
input::process_input();
let time = world.resource::<bevy::prelude::Time>().delta_secs_f64();
nodes::root::Root::send_frame_events(time);
// Wait
// we are targeting the frame after the wait
if let Some(mut state) = world.get_resource_mut::<OxrFrameState>() {
state.predicted_display_time = openxr::Time::from_nanos(
state.predicted_display_time.as_nanos() + state.predicted_display_period.as_nanos(),
);
}
let should_wait = world
.run_system_cached(should_run_frame_loop)
.unwrap_or(false);
if should_wait {
world.resource_scope::<OxrFrameWaiter, _>(|world, mut waiter| {
let state = waiter
.wait()
.inspect_err(|err| error!("failed to wait OpenXR frame: {err}"))
.ok();
if let Some(state) = state {
world.insert_resource(OxrFrameState(state));
}
});
}
tick_internal_client();
#[cfg(feature = "wayland")]

353
src/objects/input/sk_hand.rs
View File

@@ -1,15 +1,27 @@
use crate::core::client::INTERNAL_CLIENT;
use crate::nodes::OwnedNode;
use crate::nodes::fields::{Field, FieldTrait};
use crate::nodes::input::{INPUT_HANDLER_REGISTRY, InputDataType, InputHandler};
use crate::nodes::input::{Finger, INPUT_HANDLER_REGISTRY, InputDataType, InputHandler, Thumb};
use crate::nodes::{
Node,
input::{Hand, InputMethod, Joint},
spatial::Spatial,
};
use crate::objects::{ObjectHandle, SpatialRef, Tracked};
use crate::{DbusConnection, ObjectRegistryRes, PreFrameWait};
use bevy::prelude::Transform as BevyTransform;
use bevy::prelude::*;
use bevy_mod_openxr::helper_traits::{ToQuat, ToVec3};
use bevy_mod_openxr::resources::OxrFrameState;
use bevy_mod_openxr::session::OxrSession;
use bevy_mod_xr::hands::{HandBone, HandSide, XrHandBoneEntities, XrHandBoneRadius};
use bevy_mod_xr::session::{XrPreDestroySession, XrSessionCreated};
use bevy_mod_xr::spaces::{XrPrimaryReferenceSpace, XrSpaceLocationFlags};
use bevy_sk::hand::GRADIENT_TEXTURE_HANDLE;
use bevy_sk::vr_materials::PbrMaterial;
use color_eyre::eyre::Result;
use glam::{Mat4, Quat, Vec3};
use openxr::{HandJointLocation, SpaceLocationFlags};
use serde::{Deserialize, Serialize};
use stardust_xr::values::Datamap;
use std::sync::Arc;
@@ -21,15 +33,146 @@ use zbus::Connection;
use super::{CaptureManager, get_sorted_handlers};
fn convert_joint(joint: HandJoint) -> Joint {
pub struct HandPlugin;
impl Plugin for HandPlugin {
fn build(&self, app: &mut App) {
app.add_systems(PreFrameWait, update_hands);
app.add_systems(XrSessionCreated, create_trackers);
app.add_systems(XrPreDestroySession, destroy_trackers);
app.add_systems(PostUpdate, update_hand_material);
app.add_systems(Startup, setup);
}
}
fn update_hands(
mut hands: ResMut<Hands>,
session: Option<Res<OxrSession>>,
state: Option<Res<OxrFrameState>>,
ref_space: Option<Res<XrPrimaryReferenceSpace>>,
mut materials: ResMut<Assets<PbrMaterial>>,
mut joint_query: Query<(
&mut BevyTransform,
&mut XrSpaceLocationFlags,
&mut XrHandBoneRadius,
)>,
joints_query: Query<&XrHandBoneEntities>,
) {
let (Some(session), Some(state), Some(ref_space)) = (session, state, ref_space) else {
tokio::task::spawn({
let left = hands.left.tracked.clone();
let right = hands.right.tracked.clone();
async move {
left.set_tracked(false);
right.set_tracked(false);
}
});
return;
};
let get_joints = |hand: &mut SkHand| -> Option<openxr::HandJointLocations> {
let Some(tracker) = hand.tracker.as_ref() else {
hand.input.spatial.node().unwrap().set_enabled(false);
let handle = hand.tracked.clone();
tokio::task::spawn(async move {
handle.set_tracked(false);
});
return None;
};
// this won't be correct with pipelined rendering
session
.locate_hand_joints(tracker, &ref_space, state.predicted_display_time)
.inspect_err(|err| error!("Error while locating hand joints"))
.ok()
.flatten()
};
let joints_left = get_joints(&mut hands.left);
let joints_right = get_joints(&mut hands.right);
hands.left.update(joints_left.as_ref(), &mut materials);
hands.right.update(joints_right.as_ref(), &mut materials);
}
fn pinch_between(joint_1: &Joint, joint_2: &Joint) -> f32 {
const PINCH_MAX: f32 = 0.11;
const PINCH_ACTIVACTION_DISTANCE: f32 = 0.01;
let combined_radius = joint_1.radius + joint_2.radius;
let pinch_dist =
Vec3::from(joint_1.position).distance(Vec3::from(joint_2.position)) - combined_radius;
(1.0 - ((pinch_dist - PINCH_ACTIVACTION_DISTANCE) / (PINCH_MAX - PINCH_ACTIVACTION_DISTANCE)))
.clamp(0.0, 1.0)
}
fn create_trackers(session: Res<OxrSession>, mut hands: ResMut<Hands>) {
hands.left.tracker = session
.create_hand_tracker(openxr::HandEXT::LEFT)
.inspect_err(|err| error!("failed to create left hand tracker"))
.ok();
hands.right.tracker = session
.create_hand_tracker(openxr::HandEXT::RIGHT)
.inspect_err(|err| error!("failed to create right hand tracker"))
.ok();
}
fn destroy_trackers(mut hands: ResMut<Hands>) {
hands.left.tracker.take();
hands.right.tracker.take();
}
#[derive(Component)]
struct CorrectHandMaterial;
fn update_hand_material(
query: Query<
(Entity, &HandSide),
(
With<XrHandBoneEntities>,
With<MeshMaterial3d<PbrMaterial>>,
Without<CorrectHandMaterial>,
),
>,
mut cmds: Commands,
hands: Res<Hands>,
) {
for (entity, side) in &query {
let handle = match side {
HandSide::Left => hands.left.material.clone(),
HandSide::Right => hands.right.material.clone(),
};
cmds.entity(entity)
.insert(MeshMaterial3d(handle))
.insert(CorrectHandMaterial);
}
}
fn setup(
connection: Res<DbusConnection>,
mut cmds: Commands,
mut materials: ResMut<Assets<PbrMaterial>>,
) {
tokio::task::spawn({
let connection = connection.clone();
async move {
connection
.request_name("org.stardustxr.Hands")
.await
.unwrap();
}
});
cmds.insert_resource(Hands {
left: SkHand::new(&connection, HandSide::Left, &mut materials).unwrap(),
right: SkHand::new(&connection, HandSide::Right, &mut materials).unwrap(),
});
}
fn convert_joint(joint: HandJointLocation) -> Joint {
Joint {
position: Vec3::from(joint.position).into(),
rotation: Quat::from(joint.orientation).into(),
position: joint.pose.position.to_vec3().into(),
rotation: joint.pose.orientation.to_quat().into(),
radius: joint.radius,
distance: 0.0,
}
}
#[derive(Resource)]
struct Hands {
left: SkHand,
right: SkHand,
}
#[derive(Default, Deserialize, Serialize)]
struct HandDatamap {
pinch_strength: f32,
@@ -40,117 +183,158 @@ pub struct SkHand {
_node: OwnedNode,
palm_spatial: Arc<Spatial>,
palm_object: ObjectHandle<SpatialRef>,
handed: Handed,
side: HandSide,
input: Arc<InputMethod>,
capture_manager: CaptureManager,
datamap: HandDatamap,
tracked: ObjectHandle<Tracked>,
tracker: Option<openxr::HandTracker>,
captured: bool,
material: Handle<PbrMaterial>,
}
impl SkHand {
pub fn new(connection: &Connection, handed: Handed) -> Result<Self> {
pub fn new(
connection: &Connection,
side: HandSide,
materials: &mut Assets<PbrMaterial>,
) -> Result<Self> {
let (palm_spatial, palm_object) = SpatialRef::create(
connection,
&("/org/stardustxr/Hand/".to_string()
+ match handed {
Handed::Left => "left",
_ => "right",
+ match side {
HandSide::Left => "left",
HandSide::Right => "right",
} + "/palm"),
);
let tracked = Tracked::new(
connection,
&("/org/stardustxr/Hand/".to_string()
+ match handed {
Handed::Left => "left",
_ => "right",
+ match side {
HandSide::Left => "left",
HandSide::Right => "right",
}),
);
let _node = Node::generate(&INTERNAL_CLIENT, false).add_to_scenegraph_owned()?;
Spatial::add_to(&_node.0, None, Mat4::IDENTITY, false);
let node = Node::generate(&INTERNAL_CLIENT, false).add_to_scenegraph_owned()?;
Spatial::add_to(&node.0, None, Mat4::IDENTITY, false);
let hand = InputDataType::Hand(Hand {
right: handed == Handed::Right,
right: matches!(side, HandSide::Right),
..Default::default()
});
let datamap = Datamap::from_typed(HandDatamap::default())?;
let input = InputMethod::add_to(&_node.0, hand, datamap)?;
Input::hand_visible(handed, true);
let input = InputMethod::add_to(&node.0, hand, datamap)?;
let material = materials.add(PbrMaterial {
color: Srgba::new(1.0, 1.0, 1.0, 1.0).into(),
alpha_mode: AlphaMode::Blend,
use_stereokit_uvs: false,
diffuse_texture: Some(GRADIENT_TEXTURE_HANDLE),
roughness: 1.0,
..default()
});
Ok(SkHand {
_node,
_node: node,
palm_spatial,
palm_object,
handed,
side,
input,
tracked,
capture_manager: CaptureManager::default(),
datamap: Default::default(),
tracker: None,
material,
captured: false,
})
}
pub fn update(&mut self, sk: &Sk, token: &MainThreadToken, material: &mut Material) {
let sk_hand = Input::hand(self.handed);
let real_hand = Input::hand_source(self.handed) as u32 == HandSource::Articulated as u32;
if let InputDataType::Hand(hand) = &mut *self.input.data.lock() {
let input_node = self.input.spatial.node().unwrap();
input_node.set_enabled(
(real_hand || sk.get_active_display_mode() == DisplayMode::Flatscreen)
&& sk_hand.tracked.is_active(),
);
let enabled = input_node.enabled();
tokio::spawn({
// this is suboptimal since it probably allocates a fresh string every frame
let handle = self.tracked.clone();
async move {
handle.set_tracked(enabled).await;
}
fn update(
&mut self,
joints: Option<&openxr::HandJointLocations>,
materials: &mut ResMut<Assets<PbrMaterial>>,
) {
// TODO: use the hand data source ext
let real_hand = true;
let input_node = self.input.spatial.node().unwrap();
let is_tracked = real_hand
&& joints.is_some_and(|v| {
v.iter().all(|v| {
v.location_flags.contains(
SpaceLocationFlags::POSITION_VALID
| SpaceLocationFlags::POSITION_TRACKED
| SpaceLocationFlags::ORIENTATION_VALID
| SpaceLocationFlags::ORIENTATION_TRACKED,
)
})
});
if enabled {
hand.thumb.tip = convert_joint(sk_hand.fingers[0][4]);
hand.thumb.distal = convert_joint(sk_hand.fingers[0][3]);
hand.thumb.proximal = convert_joint(sk_hand.fingers[0][2]);
hand.thumb.metacarpal = convert_joint(sk_hand.fingers[0][1]);
for (finger, mut sk_finger) in [
(&mut hand.index, sk_hand.fingers[1]),
(&mut hand.middle, sk_hand.fingers[2]),
(&mut hand.ring, sk_hand.fingers[3]),
(&mut hand.little, sk_hand.fingers[4]),
] {
sk_finger[4].radius = 0.0;
finger.tip = convert_joint(sk_finger[4]);
finger.distal = convert_joint(sk_finger[3]);
finger.intermediate = convert_joint(sk_finger[2]);
finger.proximal = convert_joint(sk_finger[1]);
finger.metacarpal = convert_joint(sk_finger[0]);
}
hand.palm.position = Vec3::from(sk_hand.palm.position).into();
hand.palm.rotation = Quat::from(sk_hand.palm.orientation).into();
hand.palm.radius =
(sk_hand.fingers[2][0].radius + sk_hand.fingers[2][1].radius) * 0.5;
self.palm_spatial
.set_local_transform(Mat4::from_rotation_translation(
hand.palm.rotation.into(),
hand.palm.position.into(),
));
hand.wrist.position = Vec3::from(sk_hand.wrist.position).into();
hand.wrist.rotation = Quat::from(sk_hand.wrist.orientation).into();
hand.wrist.radius =
(sk_hand.fingers[0][0].radius + sk_hand.fingers[4][0].radius) * 0.5;
hand.elbow = None;
let hand_color = if self.capture_manager.capture.upgrade().is_none() {
Color128::new_rgb(1.0, 1.0, 1.0)
} else {
Color128::new_rgb(0.0, 1.0, 0.75)
};
material.color_tint(hand_color);
input_node.set_enabled(is_tracked);
tokio::task::spawn({
let handle = self.tracked.clone();
async move {
handle.set_tracked(is_tracked);
}
});
if is_tracked {
// cannot ever crash, is_tracked is only true of joints is some
let joints = joints.unwrap();
let new_hand = Hand {
right: matches!(self.side, HandSide::Right),
thumb: Thumb {
tip: convert_joint(joints[HandBone::ThumbTip as usize]),
distal: convert_joint(joints[HandBone::ThumbDistal as usize]),
proximal: convert_joint(joints[HandBone::ThumbProximal as usize]),
metacarpal: convert_joint(joints[HandBone::ThumbMetacarpal as usize]),
},
index: Finger {
tip: convert_joint(joints[HandBone::IndexTip as usize]),
distal: convert_joint(joints[HandBone::IndexDistal as usize]),
intermediate: convert_joint(joints[HandBone::IndexIntermediate as usize]),
proximal: convert_joint(joints[HandBone::IndexProximal as usize]),
metacarpal: convert_joint(joints[HandBone::IndexMetacarpal as usize]),
},
middle: Finger {
tip: convert_joint(joints[HandBone::MiddleTip as usize]),
distal: convert_joint(joints[HandBone::MiddleDistal as usize]),
intermediate: convert_joint(joints[HandBone::MiddleIntermediate as usize]),
proximal: convert_joint(joints[HandBone::MiddleProximal as usize]),
metacarpal: convert_joint(joints[HandBone::MiddleMetacarpal as usize]),
},
ring: Finger {
tip: convert_joint(joints[HandBone::RingTip as usize]),
distal: convert_joint(joints[HandBone::RingDistal as usize]),
intermediate: convert_joint(joints[HandBone::RingIntermediate as usize]),
proximal: convert_joint(joints[HandBone::RingProximal as usize]),
metacarpal: convert_joint(joints[HandBone::RingMetacarpal as usize]),
},
little: Finger {
tip: convert_joint(joints[HandBone::LittleTip as usize]),
distal: convert_joint(joints[HandBone::LittleDistal as usize]),
intermediate: convert_joint(joints[HandBone::LittleIntermediate as usize]),
proximal: convert_joint(joints[HandBone::LittleProximal as usize]),
metacarpal: convert_joint(joints[HandBone::LittleMetacarpal as usize]),
},
palm: convert_joint(joints[HandBone::Palm as usize]),
wrist: convert_joint(joints[HandBone::Wrist as usize]),
elbow: None,
};
self.palm_spatial
.set_local_transform(Mat4::from_rotation_translation(
new_hand.palm.rotation.into(),
new_hand.palm.position.into(),
));
self.datamap.pinch_strength = pinch_between(&new_hand.thumb.tip, &new_hand.index.tip);
// this is how stereokit calculates grab
self.datamap.grab_strength =
pinch_between(&new_hand.ring.tip, &new_hand.ring.metacarpal);
*self.input.data.lock() = InputDataType::Hand(new_hand);
*self.input.datamap.lock() = Datamap::from_typed(&self.datamap).unwrap();
let captured = self.capture_manager.capture.upgrade().is_some();
if captured && !self.captured {
materials.get_mut(&self.material).unwrap().color = Srgba::rgb(0., 1., 0.75).into();
} else if self.captured && !captured {
materials.get_mut(&self.material).unwrap().color = Srgba::rgb(1., 1.0, 1.0).into();
}
self.captured = captured;
}
self.datamap.pinch_strength = sk_hand.pinch_activation;
self.datamap.grab_strength = sk_hand.grip_activation;
*self.input.datamap.lock() = Datamap::from_typed(&self.datamap).unwrap();
let distance_calculator = |space: &Arc<Spatial>, data: &InputDataType, field: &Field| {
let InputDataType::Hand(hand) = data else {
@@ -183,11 +367,6 @@ impl SkHand {
.set_handler_order(sorted_handlers.iter().map(|(handler, _)| handler));
}
}
impl Drop for SkHand {
fn drop(&mut self) {
Input::hand_visible(self.handed, false);
}
}
fn joint_to_line_point(joint: &Joint, color: Color128) -> LinePoint {
LinePoint {

38
src/objects/mod.rs
View File

@@ -13,6 +13,7 @@ use input::{
eye_pointer::EyePointer, mouse_pointer::MousePointer, sk_controller::SkController,
sk_hand::SkHand,
};
use parking_lot::RwLock;
use play_space::PlaySpaceBounds;
use stardust_xr::schemas::dbus::object_registry::ObjectRegistry;
use std::{
@@ -34,16 +35,16 @@ enum Inputs {
XR {
controller_left: SkController,
controller_right: SkController,
hand_left: SkHand,
hand_right: SkHand,
// hand_left: SkHand,
// hand_right: SkHand,
eye_pointer: Option<EyePointer>,
},
MousePointer(MousePointer),
// Controllers((SkController, SkController)),
Hands {
left: SkHand,
right: SkHand,
},
// Hands {
// left: SkHand,
// right: SkHand,
// },
}
pub struct ServerObjects {
@@ -69,7 +70,8 @@ impl ServerObjects {
if play_space.is_some() {
let dbus_connection = connection.clone();
tokio::task::spawn(async move {
PlaySpaceBounds::create(&dbus_connection).await;
let play_space_data = Arc::new(RwLock::default());
PlaySpaceBounds::create(&dbus_connection, play_space_data).await;
dbus_connection
.request_name("org.stardustxr.PlaySpace")
.await
@@ -95,8 +97,8 @@ impl ServerObjects {
Inputs::XR {
controller_left: SkController::new(&connection, Handed::Left).unwrap(),
controller_right: SkController::new(&connection, Handed::Right).unwrap(),
hand_left: SkHand::new(&connection, Handed::Left).unwrap(),
hand_right: SkHand::new(&connection, Handed::Right).unwrap(),
// hand_left: SkHand::new(&connection, Handed::Left).unwrap(),
// hand_right: SkHand::new(&connection, Handed::Right).unwrap(),
eye_pointer: Device::has_eye_gaze()
.then(EyePointer::new)
.transpose()
@@ -166,8 +168,6 @@ impl ServerObjects {
Inputs::XR {
controller_left,
controller_right,
hand_left,
hand_right,
eye_pointer,
} => {
if !self.disable_controllers {
@@ -176,10 +176,10 @@ impl ServerObjects {
}
Input::hand_visible(Handed::Left, !self.disable_hands);
Input::hand_visible(Handed::Right, !self.disable_hands);
if !self.disable_hands {
hand_left.update(sk, token, &mut self.hand_materials[0]);
hand_right.update(sk, token, &mut self.hand_materials[1]);
}
// if !self.disable_hands {
// hand_left.update(sk, token, &mut self.hand_materials[0]);
// hand_right.update(sk, token, &mut self.hand_materials[1]);
// }
if let Some(eye_pointer) = eye_pointer {
eye_pointer.update();
}
@@ -191,10 +191,10 @@ impl ServerObjects {
// left.update(token);
// right.update(token);
// }
Inputs::Hands { left, right } => {
left.update(sk, token, &mut self.hand_materials[0]);
right.update(sk, token, &mut self.hand_materials[1]);
}
// Inputs::Hands { left, right } => {
// left.update(sk, token, &mut self.hand_materials[0]);
// right.update(sk, token, &mut self.hand_materials[1]);
// }
}
}
}

160
src/objects/play_space.rs
View File

@@ -1,12 +1,151 @@
use std::sync::Arc;
use bevy::prelude::*;
use bevy_mod_openxr::{
helper_traits::{ToQuat, ToVec3},
resources::OxrFrameState,
session::OxrSession,
};
use bevy_mod_xr::{
session::{XrPreDestroySession, XrSessionCreated},
spaces::{XrPrimaryReferenceSpace, XrReferenceSpace, XrSpace},
};
use openxr::SpaceLocationFlags;
use parking_lot::RwLock;
use stereokit_rust::system::World;
use zbus::{Connection, ObjectServer, interface};
pub struct PlaySpaceBounds;
use crate::{DbusConnection, PreFrameWait, nodes::spatial::Spatial};
use super::{ObjectHandle, SpatialRef, Tracked};
pub struct PlaySpacePlugin;
impl Plugin for PlaySpacePlugin {
fn build(&self, app: &mut App) {
app.add_systems(XrPreDestroySession, destroy_stage_space);
app.add_systems(XrSessionCreated, create_stage_space);
app.add_systems(PreFrameWait, update);
app.add_systems(Startup, setup);
}
}
fn setup(connection: Res<DbusConnection>, mut cmds: Commands) {
let (spatial, spatial_handle) = SpatialRef::create(&connection, "/org/stardustxr/PlaySpace");
// the OpenXR session might not exist quite yet
let tracked = Tracked::new(&connection, "/org/stardustxr/PlaySpace");
let dbus_connection = connection.clone();
let play_space_data = Arc::new(RwLock::default());
tokio::task::spawn({
let data = play_space_data.clone();
async move {
PlaySpaceBounds::create(&dbus_connection, data).await;
dbus_connection
.request_name("org.stardustxr.PlaySpace")
.await
.unwrap();
}
});
cmds.insert_resource(PlaySpace {
spatial,
_spatial_handle: spatial_handle,
tracked_handle: tracked,
bounds: play_space_data,
});
}
#[derive(Resource)]
struct StageSpace(XrSpace);
fn create_stage_space(session: Res<OxrSession>, mut cmds: Commands) {
let space = session
.create_reference_space(openxr::ReferenceSpaceType::STAGE, Transform::IDENTITY)
.inspect_err(|err| error!("failed to create Stage XrSpace"))
.ok();
if let Some(space) = space {
cmds.insert_resource(StageSpace(space.0));
}
}
fn destroy_stage_space(session: Res<OxrSession>, mut cmds: Commands, stage: Res<StageSpace>) {
session.destroy_space(stage.0);
cmds.remove_resource::<StageSpace>();
}
/// TODO: impl this
fn update(
session: Option<Res<OxrSession>>,
stage: Option<Res<StageSpace>>,
ref_space: Option<Res<XrPrimaryReferenceSpace>>,
play_space: Res<PlaySpace>,
state: Option<Res<OxrFrameState>>,
) {
let (Some(session), Some(stage), Some(ref_space), Some(state)) =
(session, stage, ref_space, state)
else {
play_space.bounds.write().drain(..);
tokio::task::spawn({
let handle = play_space.tracked_handle.clone();
async move {
handle.set_tracked(false);
}
});
return;
};
// this won't be correct with pipelined rendering
let location = session
.locate_space(&stage.0, &ref_space, state.predicted_display_time)
.inspect_err(|err| error!("Error while Locating OpenXR Stage Space {err}"));
if let Ok(location) = location {
let is_tracked = location.location_flags.contains(
SpaceLocationFlags::POSITION_VALID
| SpaceLocationFlags::POSITION_TRACKED
| SpaceLocationFlags::ORIENTATION_VALID
| SpaceLocationFlags::ORIENTATION_TRACKED,
);
tokio::task::spawn({
let handle = play_space.tracked_handle.clone();
async move {
handle.set_tracked(is_tracked);
}
});
if is_tracked {
play_space
.spatial
.set_local_transform(Mat4::from_rotation_translation(
location.pose.orientation.to_quat(),
location.pose.position.to_vec3(),
));
}
}
// session.reference_space_bounds_rect(openxr::ReferenceSpaceType::STAGE);
// if (World::has_bounds()
// && World::get_bounds_size().x != 0.0
// && World::get_bounds_size().y != 0.0)
// {
// let bounds = World::get_bounds_size();
// vec![
// ((bounds.x).into(), (bounds.y).into()),
// ((bounds.x).into(), (-bounds.y).into()),
// ((-bounds.x).into(), (-bounds.y).into()),
// ((-bounds.x).into(), (bounds.y).into()),
// ]
// } else {
// vec![]
// }
}
#[derive(Resource)]
pub struct PlaySpace {
spatial: Arc<Spatial>,
_spatial_handle: ObjectHandle<SpatialRef>,
tracked_handle: ObjectHandle<Tracked>,
bounds: Arc<RwLock<Vec<(f64, f64)>>>,
}
pub struct PlaySpaceBounds(Arc<RwLock<Vec<(f64, f64)>>>);
impl PlaySpaceBounds {
pub async fn create(connection: &Connection) {
pub async fn create(connection: &Connection, data: Arc<RwLock<Vec<(f64, f64)>>>) {
connection
.object_server()
.at("/org/stardustxr/PlaySpace", Self)
.at("/org/stardustxr/PlaySpace", Self(data))
.await
.unwrap();
}
@@ -15,19 +154,6 @@ impl PlaySpaceBounds {
impl PlaySpaceBounds {
#[zbus(property)]
fn bounds(&self) -> Vec<(f64, f64)> {
if (World::has_bounds()
&& World::get_bounds_size().x != 0.0
&& World::get_bounds_size().y != 0.0)
{
let bounds = World::get_bounds_size();
vec![
((bounds.x).into(), (bounds.y).into()),
((bounds.x).into(), (-bounds.y).into()),
((-bounds.x).into(), (-bounds.y).into()),
((-bounds.x).into(), (bounds.y).into()),
]
} else {
vec![]
}
self.0.read().clone()
}
}