StardustXR/server
1
0
Fork 0
Code Issues 11 Pull Requests 3 Actions Packages Projects Releases 5 Wiki Activity

refactor: very close to compiling, controllers should almost compile

Browse Source 
Signed-off-by: Schmarni <marnistromer@gmail.com>
This commit is contained in:
Schmarni
2024-12-16 06:00:29 +01:00
parent 6a23dea9b9
commit fa0eedd882
5 changed files with 373 additions and 141 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
src/bevy_plugin.rs
View File

@@ -1,6 +1,8 @@
use bevy::{
app::MainScheduleOrder, asset::embedded_asset, ecs::schedule::ScheduleLabel,
math::bounding::Aabb3d, prelude::*,
app::MainScheduleOrder,
ecs::schedule::{ExecutorKind, ScheduleLabel},
math::bounding::Aabb3d,
prelude::*,
};
use bevy_mod_openxr::session::OxrSession;
use bevy_mod_xr::session::{session_available, XrSessionCreated};
@@ -11,6 +13,11 @@ use crate::objects::Inputs;
pub struct StardustBevyPlugin;
#[derive(Resource, Deref)]
pub struct DbusConnection(pub zbus::Connection);
#[derive(ScheduleLabel, Hash, Debug, PartialEq, Eq, Clone)]
pub struct InputUpdate;
impl Plugin for StardustBevyPlugin {
fn build(&self, app: &mut App) {
app.init_schedule(StardustExtract);
@@ -19,7 +26,9 @@ impl Plugin for StardustBevyPlugin {
labels.insert(labels.len() - 2, StardustExtract.intern());
app.add_systems(Startup, spawn_camera.run_if(not(session_available)));
app.add_systems(XrSessionCreated, make_view_space);
embedded_asset!(app, "src/objects/input", "objects/input/cursor.glb");
let mut schedule = Schedule::new(InputUpdate);
schedule.set_executor_kind(ExecutorKind::Simple);
app.add_schedule(schedule);
}
}

96
src/main.rs
View File

@@ -4,15 +4,16 @@ mod core;
mod nodes;
mod objects;
pub mod oxr_render_plugin;
pub mod oxr_sync_actionset;
mod session;
#[cfg(feature = "wayland")]
mod wayland;
use crate::core::destroy_queue;
use crate::nodes::items::camera;
// use crate::nodes::items::camera;
use crate::nodes::{audio, drawable, input};
use bevy::app::{App, Startup, Update};
use bevy::app::{App, PluginGroup, Startup, Update};
use bevy::asset::{AssetServer, Handle};
use bevy::core_pipeline::Skybox;
use bevy::image::Image;
@@ -24,7 +25,7 @@ use bevy::prelude::{
use bevy::render::pipelined_rendering::PipelinedRenderingPlugin;
use bevy::time::Time;
use bevy::DefaultPlugins;
use bevy_mod_openxr::add_xr_plugins;
use bevy_mod_openxr::action_set_syncing::{OxrActionSyncingPlugin, OxrSyncActionSet};
use bevy_mod_openxr::exts::OxrExtensions;
use bevy_mod_openxr::features::overlay::{OxrOverlaySessionEvent, OxrOverlaySettings};
use bevy_mod_openxr::init::{should_run_frame_loop, OxrInitPlugin};
@@ -32,13 +33,18 @@ use bevy_mod_openxr::render::{update_cameras, OxrRenderPlugin};
use bevy_mod_openxr::resources::{OxrFrameState, OxrFrameWaiter, OxrGraphicsInfo};
use bevy_mod_openxr::session::OxrSession;
use bevy_mod_openxr::types::{AppInfo, Version};
use bevy_mod_openxr::{add_xr_plugins, openxr_session_running};
use bevy_mod_xr::session::{XrFirst, XrSessionCreated, XrSessionPlugin};
use bevy_plugin::StardustBevyPlugin;
use bevy_plugin::{DbusConnection, InputUpdate, StardustBevyPlugin};
use clap::Parser;
use core::client::Client;
use core::task;
use directories::ProjectDirs;
use nodes::audio::StardustSoundPlugin;
use nodes::drawable::lines::BevyLinesPlugin;
use nodes::drawable::model::StardustModelPlugin;
use nodes::drawable::text::StardustTextPlugin;
use objects::input::sk_controller::StardustControllerPlugin;
use objects::ServerObjects;
use once_cell::sync::OnceCell;
use openxr::OverlaySessionCreateFlagsEXTX;
@@ -52,7 +58,7 @@ use std::time::Duration;
use tokio::net::UnixListener;
use tokio::sync::Notify;
use tracing::metadata::LevelFilter;
use tracing::{debug_span, error, info};
use tracing::{debug_span, error, info, warn};
use tracing_subscriber::{fmt, prelude::*, EnvFilter};
use zbus::fdo::ObjectManager;
use zbus::Connection;
@@ -209,7 +215,9 @@ fn stereokit_loop(
object_registry: ObjectRegistry,
) {
let mut bevy_app = App::new();
let base = DefaultPlugins::build(DefaultPlugins).disable::<PipelinedRenderingPlugin>();
let base = (DefaultPlugins)
.build()
.disable::<PipelinedRenderingPlugin>();
if args.flatscreen {
bevy_app.add_plugins(base);
} else {
@@ -236,6 +244,7 @@ fn stereokit_loop(
..Default::default()
})
.disable::<OxrRenderPlugin>()
.disable::<OxrActionSyncingPlugin>()
.add_after::<XrSessionPlugin>(StardustOxrRenderPlugin),
);
if let Some(priority) = args.overlay_priority {
@@ -244,22 +253,32 @@ fn stereokit_loop(
flags: OverlaySessionCreateFlagsEXTX::EMPTY,
});
}
bevy_app.add_event::<OxrSyncActionSet>();
}
bevy_app.add_plugins(StardustBevyPlugin);
bevy_app.add_plugins(BevyLinesPlugin);
bevy_app.add_plugins((
BevyLinesPlugin,
StardustModelPlugin,
StardustTextPlugin,
StardustSoundPlugin,
StardustControllerPlugin,
));
#[derive(Resource)]
struct SkyTexture(Handle<Image>);
// Skytex/light stuff
bevy_app.add_systems(Startup, |assests: Res<AssetServer>, mut cmds: Commands| {
if let Some(sky) = project_dirs
.as_ref()
.map(|dirs| dirs.config_dir().join("skytex.hdr"))
.filter(|f| f.exists())
.map(|p| assests.load(p))
{
cmds.insert_resource(SkyTexture(sky));
}
});
bevy_app.add_systems(
Startup,
move |assests: Res<AssetServer>, mut cmds: Commands| {
if let Some(sky) = project_dirs
.as_ref()
.map(|dirs| dirs.config_dir().join("skytex.hdr"))
.filter(|f| f.exists())
.map(|p| assests.load(p))
{
cmds.insert_resource(SkyTexture(sky));
}
},
);
#[derive(Resource)]
struct RenderBackground(bool);
fn update_background(
@@ -291,6 +310,7 @@ fn stereokit_loop(
Update,
update_background.run_if(on_event::<OxrOverlaySessionEvent>),
);
bevy_app.insert_resource(DbusConnection(dbus_connection.clone()));
#[cfg(feature = "wayland")]
let mut wayland = wayland::Wayland::new().expect("Could not initialize wayland");
@@ -300,9 +320,21 @@ fn stereokit_loop(
info!("Stardust ready!");
let mut objects = ServerObjects::new(dbus_connection.clone());
fn sync_sets(session: Res<OxrSession>, mut events: EventReader<OxrSyncActionSet>) {
let sets = events
.read()
.map(|v| &v.0)
.map(openxr::ActiveActionSet::new)
.collect::<Vec<_>>();
if sets.is_empty() {
return;
}
if let Err(err) = session.sync_actions(&sets) {
warn!("error while syncing actionsets: {}", err.to_string());
}
}
let mut last_frame_delta = Duration::ZERO;
let mut sleep_duration = Duration::ZERO;
debug_span!("bevy").in_scope(|| loop {
let _span = debug_span!("Bevy step");
let _span = _span.enter();
@@ -313,33 +345,43 @@ fn stereokit_loop(
destroy_queue::clear();
let world = bevy_app.world_mut();
let session = world.remove_resource::<OxrSession>();
let time = world.get_resource::<OxrFrameState>().map(|s| {
openxr::Time::from_nanos(
let time = world.get_resource_mut::<OxrFrameState>().map(|mut s| {
let t = openxr::Time::from_nanos(
s.predicted_display_time.as_nanos() + s.predicted_display_period.as_nanos(),
)
);
s.predicted_display_time = t;
t
});
world.run_schedule(XrFirst);
if world
.run_system_cached(openxr_session_running)
.unwrap_or(true)
{
world.run_system_cached(sync_sets);
}
world.run_schedule(InputUpdate);
let session = world.remove_resource::<OxrSession>();
objects.update(session.as_deref(), time);
if let Some(session) = session {
world.insert_resource(session);
}
input::process_input();
nodes::root::Root::send_frame_events(world.resource::<Time>().delta_secs_f64());
world.run_schedule(XrFirst);
if world
.run_system_cached(should_run_frame_loop)
.unwrap_or(true)
{
let waiter = world.remove_resource::<OxrFrameWaiter>().unwrap();
let mut waiter = world.remove_resource::<OxrFrameWaiter>().unwrap();
let state = waiter.wait().unwrap();
world.insert_resource(OxrFrameState(state));
world.run_system_cached(update_cameras);
}
#[cfg(feature = "wayland")]
wayland.update();
drawable::draw(token);
bevy_app.update();
if let Some(exit) = bevy_app.should_exit() {
break;
}
audio::update();
});
#[cfg(feature = "wayland")]

21
src/nodes/items/camera.rs
View File

@@ -8,10 +8,7 @@ use crate::nodes::AspectIdentifier;
use crate::{
core::{client::Client, registry::Registry, scenegraph::MethodResponseSender},
nodes::{
drawable::{
model::{MaterialWrapper, ModelPart},
shaders::UNLIT_SHADER_BYTES,
},
drawable::{model::ModelPart, shaders::UNLIT_SHADER_BYTES},
items::TypeInfo,
spatial::{Spatial, Transform},
Message, Node,
@@ -25,20 +22,8 @@ use parking_lot::Mutex;
use stardust_xr::schemas::flex::{deserialize, serialize};
use std::sync::Arc;
use stereokit_rust::{
material::{Material, Transparency},
shader::Shader,
sk::MainThreadToken,
system::Renderer,
tex::{Tex, TexFormat, TexType},
util::Color128,
};
use tracing::error;
pub struct TexWrapper(pub Tex);
unsafe impl Send for TexWrapper {}
unsafe impl Sync for TexWrapper {}
stardust_xr_server_codegen::codegen_item_camera_protocol!();
lazy_static! {
pub(super) static ref ITEM_TYPE_INFO_CAMERA: TypeInfo = TypeInfo {
@@ -68,8 +53,8 @@ struct FrameInfo {
pub struct CameraItem {
space: Arc<Spatial>,
frame_info: Mutex<FrameInfo>,
sk_tex: OnceCell<TexWrapper>,
sk_mat: OnceCell<Arc<MaterialWrapper>>,
sk_tex: OnceCell<()>,
sk_mat: OnceCell<Arc<()>>,
applied_to: Registry<ModelPart>,
apply_to: Registry<ModelPart>,
}

11
src/nodes/items/mod.rs
View File

@@ -1,7 +1,8 @@
pub mod camera;
// TODO: reimplement with bevy
// pub mod camera;
pub mod panel;
use self::camera::CameraItem;
// use self::camera::CameraItem;
use self::panel::PanelItemTrait;
use super::alias::AliasList;
use super::fields::{Field, FIELD_ALIAS_INFO};
@@ -134,19 +135,19 @@ impl Drop for Item {
}
pub enum ItemType {
Camera(Arc<CameraItem>),
// Camera(CameraItem),
Panel(Arc<dyn PanelItemTrait>),
}
impl ItemType {
fn send_ui_item_created(&self, node: &Node, item: &Arc<Node>) {
match self {
ItemType::Camera(c) => c.send_ui_item_created(node, item),
// ItemType::Camera(c) => c.send_ui_item_created(node, item),
ItemType::Panel(p) => p.send_ui_item_created(node, item),
}
}
fn send_acceptor_item_created(&self, node: &Node, item: &Arc<Node>) {
match self {
ItemType::Camera(c) => c.send_acceptor_item_created(node, item),
// ItemType::Camera(c) => c.send_acceptor_item_created(node, item),
ItemType::Panel(p) => p.send_acceptor_item_created(node, item),
}
}

371
src/objects/input/sk_controller.rs
View File

@@ -1,5 +1,6 @@
use super::{get_sorted_handlers, CaptureManager};
use crate::{
bevy_plugin::DbusConnection,
core::client::INTERNAL_CLIENT,
nodes::{
fields::{Field, FieldTrait},
@@ -10,13 +11,34 @@ use crate::{
objects::{ObjectHandle, SpatialRef},
DefaultMaterial,
};
use bevy::{asset::Handle, prelude::Mesh};
use bevy_mod_xr::hands::HandSide;
use bevy::{
app::{App, Plugin},
asset::{embedded_asset, AssetServer, Assets, Handle},
color::LinearRgba,
gltf::GltfAssetLabel,
pbr::MeshMaterial3d,
prelude::{Children, Commands, Component, Mesh, Query, Res, ResMut, Transform},
scene::SceneRoot,
};
use bevy_mod_openxr::{
helper_traits::{ToQuat, ToVec2, ToVec3},
resources::OxrFrameState,
session::OxrSession,
spaces::{OxrSpaceExt, OxrSpaceLocationFlags},
};
use bevy_mod_xr::{
hands::HandSide,
session::XrSessionCreated,
spaces::{XrPrimaryReferenceSpace, XrSpace},
};
use color_eyre::eyre::Result;
use glam::{Mat4, Vec2, Vec3};
use once_cell::sync::OnceCell;
use openxr::{ActionSet, Posef};
use serde::{Deserialize, Serialize};
use stardust_xr::values::Datamap;
use std::sync::Arc;
use std::{ops::Deref, sync::Arc};
use tracing::error;
use zbus::Connection;
#[derive(Default, Debug, Deserialize, Serialize)]
@@ -28,96 +50,269 @@ struct ControllerDatamap {
scroll: Vec2,
}
pub struct StardustControllerPlugin;
impl Plugin for StardustControllerPlugin {
fn build(&self, app: &mut App) {
embedded_asset!(app, "src/objects/input", "cursor.glb");
app.add_systems(XrSessionCreated, spawn_controllers);
}
}
fn update_controllers(
mut mats: ResMut<Assets<DefaultMaterial>>,
mut query: Query<(&SkController, &mut Transform)>,
time: Res<OxrFrameState>,
base_space: Res<XrPrimaryReferenceSpace>,
session: ResMut<OxrSession>,
) {
for (controller, mut transform) in &mut query {
let input_node = controller.input.spatial.node().unwrap();
let location = (|| {
let location = match session.locate_space(
&XrSpace::from_raw_openxr_space(controller.space.as_raw()),
&base_space,
time.predicted_display_time,
) {
Err(err) => {
error!("issues locating controller space: {err}");
return None;
}
Ok(val) => val,
};
let flags = OxrSpaceLocationFlags(location.location_flags);
input_node.set_enabled(flags.pos_tracked() && flags.rot_tracked());
if flags.pos_valid() && flags.rot_valid() {
Some(Mat4::from_rotation_translation(
location.pose.orientation.to_quat(),
location.pose.position.to_vec3(),
))
} else {
None
}
})()
.unwrap_or(Mat4::IDENTITY);
if input_node.enabled() {
let world_transform = location;
if let Some(mat) = controller.material.get().and_then(|v| mats.get_mut(v)) {
mat.base_color = if controller.capture.is_none() {
LinearRgba::rgb(1.0, 1.0, 1.0)
} else {
LinearRgba::rgb(0.0, 1.0, 0.75)
}
.into();
}
*transform =
Transform::from_matrix(world_transform * Mat4::from_scale(Vec3::ONE * 0.02));
controller
.input
.spatial
.set_local_transform(world_transform);
}
controller.datamap.select = controller
.actions
.trigger
.state(&session, openxr::Path::NULL)
.map(|v| v.current_state)
.unwrap_or_default();
controller.datamap.grab = controller
.actions
.grip
.state(&session, openxr::Path::NULL)
.map(|v| v.current_state)
.unwrap_or_default();
controller.datamap.scroll = controller
.actions
.stick
.state(&session, openxr::Path::NULL)
.map(|v| v.current_state.to_vec2())
.unwrap_or_default();
*controller.input.datamap.lock() = Datamap::from_typed(&controller.datamap).unwrap();
// remove the capture when it's removed from captures list
if let Some(capture) = &controller.capture {
if !controller
.input
.capture_requests
.get_valid_contents()
.contains(capture)
{
controller.capture.take();
}
}
// add the capture that's the closest if we don't have one
if controller.capture.is_none() {
controller.capture = controller
.input
.capture_requests
.get_valid_contents()
.into_iter()
.map(|handler| {
(
handler.clone(),
handler
.field
.distance(&controller.input.spatial, [0.0; 3].into())
.abs(),
)
})
.reduce(|(handlers_a, distance_a), (handlers_b, distance_b)| {
if distance_a < distance_b {
(handlers_a, distance_a)
} else {
(handlers_b, distance_b)
}
})
.map(|(rx, _)| rx);
}
// make sure that if something is captured only send input to it
controller.input.captures.clear();
if let Some(capture) = &controller.capture {
controller.input.set_handler_order([capture].into_iter());
controller.input.captures.add_raw(capture);
return;
}
// send input to all the input handlers that are the closest to the ray as possible
controller.input.set_handler_order(
INPUT_HANDLER_REGISTRY
.get_valid_contents()
.into_iter()
// filter out all the disabled handlers
.filter(|handler| {
let Some(node) = handler.spatial.node() else {
return false;
};
node.enabled()
})
// filter out all the fields with disabled handlers
.filter(|handler| {
let Some(node) = handler.field.spatial.node() else {
return false;
};
node.enabled()
})
// get the unsigned distance to the handler's field (unsigned so giant fields won't always eat input)
.map(|handler| {
(
vec![handler.clone()],
handler
.field
.distance(&controller.input.spatial, [0.0; 3].into())
.abs(),
)
})
// now collect all handlers that are same distance if they're the closest
.reduce(|(mut handlers_a, distance_a), (handlers_b, distance_b)| {
if (distance_a - distance_b).abs() < 0.001 {
// distance is basically the same (within 1mm)
handlers_a.extend(handlers_b);
(handlers_a, distance_a)
} else if distance_a < distance_b {
(handlers_a, distance_a)
} else {
(handlers_b, distance_b)
}
})
.map(|(rx, _)| rx)
.unwrap_or_default()
.iter(),
);
}
}
fn spawn_controllers(
connection: Res<DbusConnection>,
asset_server: Res<AssetServer>,
session: Res<OxrSession>,
mut cmds: Commands,
) {
let handle = asset_server.load(GltfAssetLabel::Scene(0).from_asset("embedded://cursor.glb"));
for handed in [HandSide::Left, HandSide::Right] {
let side = match handed {
HandSide::Left => "left",
HandSide::Right => "right",
};
let (spatial, object_handle) = SpatialRef::create(
&connection,
&("/org/stardustxr/Controller/".to_string() + side),
);
let tip = InputDataType::Tip(Tip::default());
let Ok(input) = (|| -> color_eyre::Result<Arc<InputMethod>> {
Ok(InputMethod::add_to(
&spatial.node().unwrap(),
tip,
Datamap::from_typed(ControllerDatamap::default())?,
)?)
})() else {
continue;
};
let actions = {
let set = session
.instance()
.create_action_set(
&format!("controller-{side}"),
&format!("{side} controller"),
0,
)
.unwrap();
Actions {
set: set.clone(),
trigger: set
.create_action(&format!("trigger-{side}"), &format!("{side} trigger"), &[])
.unwrap(),
grip: set
.create_action(&format!("grip-{side}"), &format!("{side} grip"), &[])
.unwrap(),
stick: set
.create_action(&format!("stick-{side}"), &format!("{side} stick"), &[])
.unwrap(),
pose: set
.create_action(&format!("pose-{side}"), &format!("{side} pose"), &[])
.unwrap(),
}
};
cmds.spawn((
SceneRoot(handle.clone()),
SkController {
object_handle,
input,
handed,
material: OnceCell::new(),
capture: None,
datamap: Default::default(),
space: actions
.pose
.create_space(
session.deref().deref().clone(),
openxr::Path::NULL,
Posef::IDENTITY,
)
.unwrap(),
actions,
},
));
}
}
#[derive(Component)]
#[require(Transform)]
pub struct SkController {
object_handle: ObjectHandle<SpatialRef>,
input: Arc<InputMethod>,
handed: HandSide,
color: bevy::color::Color,
material: OnceCell<Handle<DefaultMaterial>>,
capture: Option<Arc<InputHandler>>,
datamap: ControllerDatamap,
space: openxr::Space,
actions: Actions,
}
impl SkController {
pub fn new(connection: &Connection, handed: Handed) -> Result<Self> {
let (spatial, object_handle) = SpatialRef::create(
connection,
&("/org/stardustxr/Controller/".to_string()
+ match handed {
Handed::Left => "left",
_ => "right",
}),
);
let model = Model::copy(Model::from_memory(
"cursor.glb",
include_bytes!("cursor.glb"),
None,
)?);
let model_nodes = model.get_nodes();
let mut model_node = model_nodes.visuals().next().unwrap();
let material = Material::copy(&model_node.get_material().unwrap());
model_node.material(&material);
let tip = InputDataType::Tip(Tip::default());
let input = InputMethod::add_to(
&spatial.node().unwrap(),
tip,
Datamap::from_typed(ControllerDatamap::default())?,
)?;
Ok(SkController {
object_handle,
input,
handed,
model,
material,
capture_manager: CaptureManager::default(),
datamap: Default::default(),
})
}
pub fn update(&mut self, token: &MainThreadToken) {
let controller = Input::controller(self.handed);
let input_node = self.input.spatial.node().unwrap();
input_node.set_enabled(controller.tracked.is_active());
if input_node.enabled() {
let world_transform = Mat4::from_rotation_translation(
controller.aim.orientation.into(),
controller.aim.position.into(),
);
self.material
.color_tint(if self.capture_manager.capture.is_none() {
Color128::new_rgb(1.0, 1.0, 1.0)
} else {
Color128::new_rgb(0.0, 1.0, 0.75)
});
self.model.draw(
token,
world_transform * Mat4::from_scale(Vec3::ONE * 0.02),
None,
None,
);
self.input.spatial.set_local_transform(world_transform);
}
self.datamap = ControllerDatamap {
select: controller.trigger,
middle: controller.stick_click.is_active() as u32 as f32,
context: controller.is_x2_pressed() as u32 as f32,
grab: controller.grip,
scroll: controller.stick.into(),
};
*self.input.datamap.lock() = Datamap::from_typed(&self.datamap).unwrap();
let distance_calculator = |space: &Arc<Spatial>, _data: &InputDataType, field: &Field| {
Some(field.distance(space, [0.0; 3].into()).abs())
};
self.capture_manager.update_capture(&self.input);
self.capture_manager
.set_new_capture(&self.input, distance_calculator);
self.capture_manager.apply_capture(&self.input);
if self.capture_manager.capture.is_some() {
return;
}
let sorted_handlers = get_sorted_handlers(&self.input, distance_calculator);
self.input.set_handler_order(sorted_handlers.iter());
}
struct Actions {
set: openxr::ActionSet,
trigger: openxr::Action<f32>,
grip: openxr::Action<f32>,
stick: openxr::Action<openxr::Vector2f>,
pose: openxr::Action<openxr::Posef>,
}