From b16ba65fd307c4476d2a155448cc6a2906e4bb87 Mon Sep 17 00:00:00 2001 From: Schmarni Date: Sat, 28 Jun 2025 20:24:02 +0200 Subject: [PATCH] feat: implement audio! thats all nodes! Signed-off-by: Schmarni --- Cargo.lock | 66 ++++++++++++++++++++ Cargo.toml | 2 +- src/main.rs | 4 +- src/nodes/audio.rs | 119 +++++++++++++++++++++++------------- src/nodes/drawable/lines.rs | 7 ++- src/nodes/drawable/model.rs | 2 +- src/nodes/drawable/text.rs | 26 +++++--- 7 files changed, 170 insertions(+), 56 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3404016..47c4c5e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2656,6 +2656,15 @@ dependencies = [ "syn 2.0.103", ] +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + [[package]] name = "endi" version = "1.1.0" @@ -3367,6 +3376,12 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfa686283ad6dd069f105e5ab091b04c62850d3e4cf5d67debad1933f55023df" +[[package]] +name = "hound" +version = "3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62adaabb884c94955b19907d60019f4e145d091c75345379e70d1ee696f7854f" + [[package]] name = "http" version = "1.3.1" @@ -5308,7 +5323,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7ceb6607dd738c99bc8cb28eff249b7cd5c8ec88b9db96c0608c1480d140fb1" dependencies = [ "cpal", + "hound", "lewton", + "symphonia", ] [[package]] @@ -5937,6 +5954,55 @@ dependencies = [ "zeno", ] +[[package]] +name = "symphonia" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "815c942ae7ee74737bb00f965fa5b5a2ac2ce7b6c01c0cc169bbeaf7abd5f5a9" +dependencies = [ + "lazy_static", + "symphonia-bundle-mp3", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-bundle-mp3" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c01c2aae70f0f1fb096b6f0ff112a930b1fb3626178fba3ae68b09dce71706d4" +dependencies = [ + "lazy_static", + "log", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-core" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "798306779e3dc7d5231bd5691f5a813496dc79d3f56bf82e25789f2094e022c3" +dependencies = [ + "arrayvec", + "bitflags 1.3.2", + "bytemuck", + "lazy_static", + "log", +] + +[[package]] +name = "symphonia-metadata" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc622b9841a10089c5b18e99eb904f4341615d5aa55bbf4eedde1be721a4023c" +dependencies = [ + "encoding_rs", + "lazy_static", + "log", + "symphonia-core", +] + [[package]] name = "syn" version = "1.0.109" diff --git a/Cargo.toml b/Cargo.toml index 392f90a..1991456 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -79,7 +79,7 @@ mint = "0.5.9" tokio = { version = "1.39.2", features = ["rt-multi-thread", "signal", "time"] } # bevy -bevy = { version = "0.16", features = ["wayland", "bevy_remote"] } +bevy = { version = "0.16", features = ["wayland", "bevy_remote", "mp3", "wav"] } bevy_mod_xr = "0.3" bevy_mod_openxr = "0.3" # bevy_mod_meshtext.git = "https://github.com/Schmarni-Dev/bevy_mod_meshtext" diff --git a/src/main.rs b/src/main.rs index 1f88cac..cf75b03 100644 --- a/src/main.rs +++ b/src/main.rs @@ -42,6 +42,7 @@ use bevy_mod_xr::camera::XrProjection; use bevy_mod_xr::hand_debug_gizmos::HandGizmosPlugin; use bevy_mod_xr::session::{XrFirst, XrHandleEvents}; use clap::Parser; +use nodes::audio::AudioNodePlugin; use core::client::{Client, tick_internal_client}; use core::entity_handle::EntityHandlePlugin; use core::task; @@ -394,6 +395,7 @@ fn bevy_loop( ModelNodePlugin, TextNodePlugin, LinesNodePlugin, + AudioNodePlugin, )); // object plugins app.add_plugins((PlaySpacePlugin, HandPlugin, ControllerPlugin)); @@ -576,7 +578,7 @@ fn stereokit_loop( #[cfg(feature = "wayland")] wayland.update(); drawable::draw(token); - audio::update(); + // audio::update(); } info!("Cleanly shut down StereoKit"); diff --git a/src/nodes/audio.rs b/src/nodes/audio.rs index d048dd8..7c165ba 100644 --- a/src/nodes/audio.rs +++ b/src/nodes/audio.rs @@ -1,26 +1,93 @@ +use super::spatial::SpatialNode; use super::{Aspect, AspectIdentifier, Node}; use crate::core::client::Client; -use crate::core::destroy_queue; +use crate::core::entity_handle::EntityHandle; use crate::core::error::Result; use crate::core::registry::Registry; use crate::core::resource::get_resource_file; use crate::nodes::spatial::{SPATIAL_ASPECT_ALIAS_INFO, Spatial, Transform}; +use bevy::audio::{PlaybackMode, Volume}; +use bevy_mod_openxr::session::OxrSession; +use bevy_mod_xr::session::{XrPreDestroySession, XrSessionCreated}; +use bevy_mod_xr::spaces::XrSpace; use color_eyre::eyre::eyre; -use glam::{Vec4Swizzles, vec3}; use parking_lot::Mutex; use stardust_xr::values::ResourceID; -use std::ops::DerefMut; +use bevy::prelude::*; +use bevy::transform::components::Transform as BevyTransform; use std::sync::{Arc, OnceLock}; use std::{ffi::OsStr, path::PathBuf}; -use stereokit_rust::sound::{Sound as SkSound, SoundInst}; -use bevy::prelude::*; pub struct AudioNodePlugin; impl Plugin for AudioNodePlugin { - fn build(&self, app: &mut App) { - todo!() - } + fn build(&self, app: &mut App) { + app.add_systems(Update, update_sound_event); + app.add_systems(XrSessionCreated, spawn_hmd_audio_listener); + app.add_systems(XrPreDestroySession, despawn_hmd_audio_listener); + } +} + +fn despawn_hmd_audio_listener(mut cmds: Commands, session: Res, res: Res) { + cmds.remove_resource::(); + cmds.entity(res.0).despawn(); + _ = session.destroy_space(res.1); +} + +fn spawn_hmd_audio_listener(mut cmds: Commands, session: Res) { + let space = session + .create_reference_space(openxr::ReferenceSpaceType::VIEW, BevyTransform::IDENTITY) + .unwrap(); + let listener = cmds + .spawn(( + Name::new("HMD audio listener"), + space.0, + SpatialListener::new(0.2), + )) + .id(); + cmds.insert_resource(HmdListener(listener, space.0)); +} +#[derive(Resource)] +struct HmdListener(Entity, XrSpace); +fn update_sound_event( + mut cmds: Commands, + sinks: Query<&SpatialAudioSink>, + asset_server: Res, +) { + for sound in SOUND_REGISTRY.get_valid_contents() { + if sound.entity.get().is_none() { + let handle = asset_server.load(sound.pending_audio_path.as_path()); + sound + .entity + .set( + cmds.spawn(( + Name::new("Audio Node"), + SpatialNode(Arc::downgrade(&sound.spatial)), + AudioPlayer::new(handle), + PlaybackSettings { + mode: PlaybackMode::Once, + volume: Volume::Linear(sound.volume), + speed: 1.0, + paused: true, + muted: false, + spatial: true, + spatial_scale: None, + }, + )) + .id() + .into(), + ) + .unwrap(); + } + if let Some(sink) = sound.entity.get().and_then(|e| sinks.get(e.0).ok()) { + if sound.play.lock().take().is_some() { + sink.play(); + } + if sound.stop.lock().take().is_some() { + sink.stop(); + } + } + } } static SOUND_REGISTRY: Registry = Registry::new(); @@ -31,8 +98,7 @@ pub struct Sound { volume: f32, pending_audio_path: PathBuf, - sk_sound: OnceLock, - instance: Mutex>, + entity: OnceLock, stop: Mutex>, play: Mutex>, } @@ -48,8 +114,7 @@ impl Sound { spatial: node.get_aspect::().unwrap().clone(), volume: 1.0, pending_audio_path, - sk_sound: OnceLock::new(), - instance: Mutex::new(None), + entity: OnceLock::new(), stop: Mutex::new(None), play: Mutex::new(None), }; @@ -57,24 +122,6 @@ impl Sound { node.add_aspect_raw(sound_arc.clone()); Ok(sound_arc) } - - fn update(&self) { - let sound = self - .sk_sound - .get_or_init(|| SkSound::from_file(self.pending_audio_path.clone()).unwrap()); - if self.stop.lock().take().is_some() { - if let Some(instance) = self.instance.lock().take() { - instance.stop(); - } - } - if self.instance.lock().is_none() && self.play.lock().take().is_some() { - let instance = sound.play(vec3(0.0, 0.0, 0.0), Some(self.volume)); - self.instance.lock().replace(instance); - } - if let Some(instance) = self.instance.lock().deref_mut() { - instance.position(self.spatial.global_transform().w_axis.xyz()); - } - } } impl AspectIdentifier for Sound { impl_aspect_for_sound_aspect_id! {} @@ -96,22 +143,10 @@ impl SoundAspect for Sound { } impl Drop for Sound { fn drop(&mut self) { - if let Some(instance) = self.instance.lock().take() { - instance.stop(); - } - if let Some(sk_sound) = self.sk_sound.take() { - destroy_queue::add(sk_sound); - } SOUND_REGISTRY.remove(self); } } -pub fn update() { - for sound in SOUND_REGISTRY.get_valid_contents() { - sound.update() - } -} - impl InterfaceAspect for Interface { #[doc = "Create a sound node. WAV and MP3 are supported."] fn create_sound( diff --git a/src/nodes/drawable/lines.rs b/src/nodes/drawable/lines.rs index 33e63cd..5f3a415 100644 --- a/src/nodes/drawable/lines.rs +++ b/src/nodes/drawable/lines.rs @@ -155,7 +155,10 @@ fn build_line_mesh( match lines.entity.get() { Some(e) => cmds.entity(**e), None => { - let e = cmds.spawn(SpatialNode(Arc::downgrade(&lines.spatial))); + let e = cmds.spawn(( + Name::new("LinesNode"), + SpatialNode(Arc::downgrade(&lines.spatial)), + )); _ = lines.entity.set(e.id().into()); e } @@ -223,7 +226,6 @@ impl Lines { .unwrap_or_default() }); - info!("line::add_to"); let lines = LINES_REGISTRY.add(Lines { spatial: node.get_aspect::()?.clone(), data: Mutex::new(lines), @@ -238,7 +240,6 @@ impl Lines { } impl LinesAspect for Lines { fn set_lines(node: Arc, _calling_client: Arc, lines: Vec) -> Result<()> { - info!("set_lines"); let lines_aspect = node.get_aspect::()?; *lines_aspect.data.lock() = lines; lines_aspect.gen_mesh.store(true, Ordering::Relaxed); diff --git a/src/nodes/drawable/model.rs b/src/nodes/drawable/model.rs index 38fbce3..1b61e0d 100644 --- a/src/nodes/drawable/model.rs +++ b/src/nodes/drawable/model.rs @@ -68,6 +68,7 @@ fn load_models( let handle = asset_server.load(GltfAssetLabel::Scene(0).from_asset(path)); let entity = cmds .spawn(( + Name::new("ModelNode"), SceneRoot(handle), ModelNode(Arc::downgrade(&model)), SpatialNode(Arc::downgrade(&model.spatial)), @@ -394,7 +395,6 @@ impl MaterialParameter { else { return; }; - info!(texture_param = parameter_name, path = ?texture_path); let handle = asset_server.load(texture_path); match parameter_name { "diffuse" => mat.diffuse_texture = Some(handle), diff --git a/src/nodes/drawable/text.rs b/src/nodes/drawable/text.rs index a701625..fff3d59 100644 --- a/src/nodes/drawable/text.rs +++ b/src/nodes/drawable/text.rs @@ -1,9 +1,17 @@ use crate::{ core::{ - bevy_channel::{BevyChannel, BevyChannelReader}, client::Client, color::ColorConvert, entity_handle::EntityHandle, error::Result, registry::Registry, resource::get_resource_file + bevy_channel::{BevyChannel, BevyChannelReader}, + client::Client, + color::ColorConvert, + entity_handle::EntityHandle, + error::Result, + registry::Registry, + resource::get_resource_file, }, nodes::{ - drawable::XAlign, spatial::{Spatial, SpatialNode}, Node + Node, + drawable::XAlign, + spatial::{Spatial, SpatialNode}, }, }; use bevy::{platform::collections::HashMap, prelude::*}; @@ -95,9 +103,8 @@ fn spawn_text( ); let max_width = style.bounds.as_ref().map(|v| v.bounds.x); let max_height = style.bounds.as_ref().map(|v| v.bounds.x); - let (width, height) = + let (width, _height) = text_glyphs.measure(max_width, max_height, &mut font_settings.font_system); - info!(width, height, ?style.text_align_x); let meshes = generate_meshes( bevy_mesh_text_3d::InputText::Simple { text: text_string, @@ -134,9 +141,9 @@ fn spawn_text( else { continue; }; - let dist = meshes.iter().fold(f32::MAX, |dist, v| { - dist.min(v.transform.translation.x) - }); + let dist = meshes + .iter() + .fold(f32::MAX, |dist, v| dist.min(v.transform.translation.x)); // TODO: text align let letters = meshes .into_iter() @@ -162,7 +169,10 @@ fn spawn_text( }) .collect::>(); let entity = cmds - .spawn((SpatialNode(Arc::downgrade(&text.spatial)),)) + .spawn(( + Name::new("TextNode"), + SpatialNode(Arc::downgrade(&text.spatial)), + )) .add_children(&letters) .id(); text.entity.lock().replace(EntityHandle(entity));