feat: implement audio! thats all nodes!

Signed-off-by: Schmarni <marnistromer@gmail.com>
2025-06-28 20:24:02 +02:00
parent 600eab9d2a
commit c93036278f
7 changed files with 170 additions and 56 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2656,6 +2656,15 @@ dependencies = [
 "syn 2.0.103",
 ]

+[[package]]
+name = "encoding_rs"
+version = "0.8.35"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3"
+dependencies = [
+ "cfg-if",
+]
+
 [[package]]
 name = "endi"
 version = "1.1.0"
@@ -3367,6 +3376,12 @@ version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dfa686283ad6dd069f105e5ab091b04c62850d3e4cf5d67debad1933f55023df"

+[[package]]
+name = "hound"
+version = "3.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "62adaabb884c94955b19907d60019f4e145d091c75345379e70d1ee696f7854f"
+
 [[package]]
 name = "http"
 version = "1.3.1"
@@ -5308,7 +5323,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e7ceb6607dd738c99bc8cb28eff249b7cd5c8ec88b9db96c0608c1480d140fb1"
 dependencies = [
 "cpal",
+ "hound",
 "lewton",
+ "symphonia",
 ]

 [[package]]
@@ -5937,6 +5954,55 @@ dependencies = [
 "zeno",
 ]

+[[package]]
+name = "symphonia"
+version = "0.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "815c942ae7ee74737bb00f965fa5b5a2ac2ce7b6c01c0cc169bbeaf7abd5f5a9"
+dependencies = [
+ "lazy_static",
+ "symphonia-bundle-mp3",
+ "symphonia-core",
+ "symphonia-metadata",
+]
+
+[[package]]
+name = "symphonia-bundle-mp3"
+version = "0.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c01c2aae70f0f1fb096b6f0ff112a930b1fb3626178fba3ae68b09dce71706d4"
+dependencies = [
+ "lazy_static",
+ "log",
+ "symphonia-core",
+ "symphonia-metadata",
+]
+
+[[package]]
+name = "symphonia-core"
+version = "0.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "798306779e3dc7d5231bd5691f5a813496dc79d3f56bf82e25789f2094e022c3"
+dependencies = [
+ "arrayvec",
+ "bitflags 1.3.2",
+ "bytemuck",
+ "lazy_static",
+ "log",
+]
+
+[[package]]
+name = "symphonia-metadata"
+version = "0.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bc622b9841a10089c5b18e99eb904f4341615d5aa55bbf4eedde1be721a4023c"
+dependencies = [
+ "encoding_rs",
+ "lazy_static",
+ "log",
+ "symphonia-core",
+]
+
 [[package]]
 name = "syn"
 version = "1.0.109"
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -79,7 +79,7 @@ mint = "0.5.9"
 tokio = { version = "1.39.2", features = ["rt-multi-thread", "signal", "time"] }

 # bevy
-bevy = { version = "0.16", features = ["wayland", "bevy_remote"] }
+bevy = { version = "0.16", features = ["wayland", "bevy_remote", "mp3", "wav"] }
 bevy_mod_xr = "0.3"
 bevy_mod_openxr = "0.3"
 # bevy_mod_meshtext.git = "https://github.com/Schmarni-Dev/bevy_mod_meshtext"
--- a/src/main.rs
+++ b/src/main.rs
@@ -42,6 +42,7 @@ use bevy_mod_xr::camera::XrProjection;
 use bevy_mod_xr::hand_debug_gizmos::HandGizmosPlugin;
 use bevy_mod_xr::session::{XrFirst, XrHandleEvents};
 use clap::Parser;
+use nodes::audio::AudioNodePlugin;
 use core::client::{Client, tick_internal_client};
 use core::entity_handle::EntityHandlePlugin;
 use core::task;
@@ -394,6 +395,7 @@ fn bevy_loop(
 		ModelNodePlugin,
 		TextNodePlugin,
 		LinesNodePlugin,
+		AudioNodePlugin,
 	));
 	// object plugins
 	app.add_plugins((PlaySpacePlugin, HandPlugin, ControllerPlugin));
@@ -576,7 +578,7 @@ fn stereokit_loop(
 		#[cfg(feature = "wayland")]
 		wayland.update();
 		drawable::draw(token);
-		audio::update();
+		// audio::update();
 	}

 	info!("Cleanly shut down StereoKit");
--- a/src/nodes/audio.rs
+++ b/src/nodes/audio.rs
@@ -1,26 +1,93 @@
+use super::spatial::SpatialNode;
 use super::{Aspect, AspectIdentifier, Node};
 use crate::core::client::Client;
-use crate::core::destroy_queue;
+use crate::core::entity_handle::EntityHandle;
 use crate::core::error::Result;
 use crate::core::registry::Registry;
 use crate::core::resource::get_resource_file;
 use crate::nodes::spatial::{SPATIAL_ASPECT_ALIAS_INFO, Spatial, Transform};
+use bevy::audio::{PlaybackMode, Volume};
+use bevy_mod_openxr::session::OxrSession;
+use bevy_mod_xr::session::{XrPreDestroySession, XrSessionCreated};
+use bevy_mod_xr::spaces::XrSpace;
 use color_eyre::eyre::eyre;
-use glam::{Vec4Swizzles, vec3};
 use parking_lot::Mutex;
 use stardust_xr::values::ResourceID;

-use std::ops::DerefMut;
+use bevy::prelude::*;
+use bevy::transform::components::Transform as BevyTransform;
 use std::sync::{Arc, OnceLock};
 use std::{ffi::OsStr, path::PathBuf};
-use stereokit_rust::sound::{Sound as SkSound, SoundInst};
-use bevy::prelude::*;

 pub struct AudioNodePlugin;
 impl Plugin for AudioNodePlugin {
-    fn build(&self, app: &mut App) {
-        todo!()
-    }
+	fn build(&self, app: &mut App) {
+		app.add_systems(Update, update_sound_event);
+		app.add_systems(XrSessionCreated, spawn_hmd_audio_listener);
+		app.add_systems(XrPreDestroySession, despawn_hmd_audio_listener);
+	}
+}
+
+fn despawn_hmd_audio_listener(mut cmds: Commands, session: Res<OxrSession>, res: Res<HmdListener>) {
+	cmds.remove_resource::<HmdListener>();
+	cmds.entity(res.0).despawn();
+	_ = session.destroy_space(res.1);
+}
+
+fn spawn_hmd_audio_listener(mut cmds: Commands, session: Res<OxrSession>) {
+	let space = session
+		.create_reference_space(openxr::ReferenceSpaceType::VIEW, BevyTransform::IDENTITY)
+		.unwrap();
+	let listener = cmds
+		.spawn((
+			Name::new("HMD audio listener"),
+			space.0,
+			SpatialListener::new(0.2),
+		))
+		.id();
+	cmds.insert_resource(HmdListener(listener, space.0));
+}
+#[derive(Resource)]
+struct HmdListener(Entity, XrSpace);
+fn update_sound_event(
+	mut cmds: Commands,
+	sinks: Query<&SpatialAudioSink>,
+	asset_server: Res<AssetServer>,
+) {
+	for sound in SOUND_REGISTRY.get_valid_contents() {
+		if sound.entity.get().is_none() {
+			let handle = asset_server.load(sound.pending_audio_path.as_path());
+			sound
+				.entity
+				.set(
+					cmds.spawn((
+						Name::new("Audio Node"),
+						SpatialNode(Arc::downgrade(&sound.spatial)),
+						AudioPlayer::new(handle),
+						PlaybackSettings {
+							mode: PlaybackMode::Once,
+							volume: Volume::Linear(sound.volume),
+							speed: 1.0,
+							paused: true,
+							muted: false,
+							spatial: true,
+							spatial_scale: None,
+						},
+					))
+					.id()
+					.into(),
+				)
+				.unwrap();
+		}
+		if let Some(sink) = sound.entity.get().and_then(|e| sinks.get(e.0).ok()) {
+			if sound.play.lock().take().is_some() {
+				sink.play();
+			}
+			if sound.stop.lock().take().is_some() {
+				sink.stop();
+			}
+		}
+	}
 }

 static SOUND_REGISTRY: Registry<Sound> = Registry::new();
@@ -31,8 +98,7 @@ pub struct Sound {

 	volume: f32,
 	pending_audio_path: PathBuf,
-	sk_sound: OnceLock<SkSound>,
-	instance: Mutex<Option<SoundInst>>,
+	entity: OnceLock<EntityHandle>,
 	stop: Mutex<Option<()>>,
 	play: Mutex<Option<()>>,
 }
@@ -48,8 +114,7 @@ impl Sound {
 			spatial: node.get_aspect::<Spatial>().unwrap().clone(),
 			volume: 1.0,
 			pending_audio_path,
-			sk_sound: OnceLock::new(),
-			instance: Mutex::new(None),
+			entity: OnceLock::new(),
 			stop: Mutex::new(None),
 			play: Mutex::new(None),
 		};
@@ -57,24 +122,6 @@ impl Sound {
 		node.add_aspect_raw(sound_arc.clone());
 		Ok(sound_arc)
 	}
-
-	fn update(&self) {
-		let sound = self
-			.sk_sound
-			.get_or_init(|| SkSound::from_file(self.pending_audio_path.clone()).unwrap());
-		if self.stop.lock().take().is_some() {
-			if let Some(instance) = self.instance.lock().take() {
-				instance.stop();
-			}
-		}
-		if self.instance.lock().is_none() && self.play.lock().take().is_some() {
-			let instance = sound.play(vec3(0.0, 0.0, 0.0), Some(self.volume));
-			self.instance.lock().replace(instance);
-		}
-		if let Some(instance) = self.instance.lock().deref_mut() {
-			instance.position(self.spatial.global_transform().w_axis.xyz());
-		}
-	}
 }
 impl AspectIdentifier for Sound {
 	impl_aspect_for_sound_aspect_id! {}
@@ -96,22 +143,10 @@ impl SoundAspect for Sound {
 }
 impl Drop for Sound {
 	fn drop(&mut self) {
-		if let Some(instance) = self.instance.lock().take() {
-			instance.stop();
-		}
-		if let Some(sk_sound) = self.sk_sound.take() {
-			destroy_queue::add(sk_sound);
-		}
 		SOUND_REGISTRY.remove(self);
 	}
 }

-pub fn update() {
-	for sound in SOUND_REGISTRY.get_valid_contents() {
-		sound.update()
-	}
-}
-
 impl InterfaceAspect for Interface {
 	#[doc = "Create a sound node. WAV and MP3 are supported."]
 	fn create_sound(
--- a/src/nodes/drawable/lines.rs
+++ b/src/nodes/drawable/lines.rs
@@ -155,7 +155,10 @@ fn build_line_mesh(
 		match lines.entity.get() {
 			Some(e) => cmds.entity(**e),
 			None => {
-				let e = cmds.spawn(SpatialNode(Arc::downgrade(&lines.spatial)));
+				let e = cmds.spawn((
+					Name::new("LinesNode"),
+					SpatialNode(Arc::downgrade(&lines.spatial)),
+				));
 				_ = lines.entity.set(e.id().into());
 				e
 			}
@@ -223,7 +226,6 @@ impl Lines {
 					.unwrap_or_default()
 			});

-		info!("line::add_to");
 		let lines = LINES_REGISTRY.add(Lines {
 			spatial: node.get_aspect::<Spatial>()?.clone(),
 			data: Mutex::new(lines),
@@ -238,7 +240,6 @@ impl Lines {
 }
 impl LinesAspect for Lines {
 	fn set_lines(node: Arc<Node>, _calling_client: Arc<Client>, lines: Vec<Line>) -> Result<()> {
-		info!("set_lines");
 		let lines_aspect = node.get_aspect::<Lines>()?;
 		*lines_aspect.data.lock() = lines;
 		lines_aspect.gen_mesh.store(true, Ordering::Relaxed);
--- a/src/nodes/drawable/model.rs
+++ b/src/nodes/drawable/model.rs
@@ -68,6 +68,7 @@ fn load_models(
 		let handle = asset_server.load(GltfAssetLabel::Scene(0).from_asset(path));
 		let entity = cmds
 			.spawn((
+				Name::new("ModelNode"),
 				SceneRoot(handle),
 				ModelNode(Arc::downgrade(&model)),
 				SpatialNode(Arc::downgrade(&model.spatial)),
@@ -394,7 +395,6 @@ impl MaterialParameter {
 				else {
 					return;
 				};
-				info!(texture_param = parameter_name, path = ?texture_path);
 				let handle = asset_server.load(texture_path);
 				match parameter_name {
 					"diffuse" => mat.diffuse_texture = Some(handle),
--- a/src/nodes/drawable/text.rs
+++ b/src/nodes/drawable/text.rs
@@ -1,9 +1,17 @@
 use crate::{
 	core::{
-		bevy_channel::{BevyChannel, BevyChannelReader}, client::Client, color::ColorConvert, entity_handle::EntityHandle, error::Result, registry::Registry, resource::get_resource_file
+		bevy_channel::{BevyChannel, BevyChannelReader},
+		client::Client,
+		color::ColorConvert,
+		entity_handle::EntityHandle,
+		error::Result,
+		registry::Registry,
+		resource::get_resource_file,
 	},
 	nodes::{
-		drawable::XAlign, spatial::{Spatial, SpatialNode}, Node
+		Node,
+		drawable::XAlign,
+		spatial::{Spatial, SpatialNode},
 	},
 };
 use bevy::{platform::collections::HashMap, prelude::*};
@@ -95,9 +103,8 @@ fn spawn_text(
 		);
 		let max_width = style.bounds.as_ref().map(|v| v.bounds.x);
 		let max_height = style.bounds.as_ref().map(|v| v.bounds.x);
-		let (width, height) =
+		let (width, _height) =
 			text_glyphs.measure(max_width, max_height, &mut font_settings.font_system);
-		info!(width, height, ?style.text_align_x);
 		let meshes = generate_meshes(
 			bevy_mesh_text_3d::InputText::Simple {
 				text: text_string,
@@ -134,9 +141,9 @@ fn spawn_text(
 		else {
 			continue;
 		};
-		let dist = meshes.iter().fold(f32::MAX, |dist, v| {
-			dist.min(v.transform.translation.x)
-		});
+		let dist = meshes
+			.iter()
+			.fold(f32::MAX, |dist, v| dist.min(v.transform.translation.x));
 		// TODO: text align
 		let letters = meshes
 			.into_iter()
@@ -162,7 +169,10 @@ fn spawn_text(
 			})
 			.collect::<Vec<_>>();
 		let entity = cmds
-			.spawn((SpatialNode(Arc::downgrade(&text.spatial)),))
+			.spawn((
+				Name::new("TextNode"),
+				SpatialNode(Arc::downgrade(&text.spatial)),
+			))
 			.add_children(&letters)
 			.id();
 		text.entity.lock().replace(EntityHandle(entity));