feat(wayland): implement shm ontop of dmabuf

Signed-off-by: Schmarni <marnistromer@gmail.com>

Cargo.lock

@@ -6284,8 +6284,7 @@ dependencies = [
 [[package]]
 name = "vulkano"
 version = "0.35.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08840c2b51759a6f88f26f5ea378bc8b5c199a5b4760ddda292304be087249c4"
+source = "git+https://github.com/Schmarni-Dev/vulkano?branch=0_35_dmabuf_fixes#1fd43557d3acb3ddd2721f61b6939ab636ecd17a"
 dependencies = [
  "ash",
  "bytemuck",
@@ -6316,8 +6315,7 @@ dependencies = [
 [[package]]
 name = "vulkano-macros"
 version = "0.35.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1dc929c42c9336fd082079ac3ea30126e4a0dfe36fd2e2b3581303f7d140d20f"
+source = "git+https://github.com/Schmarni-Dev/vulkano?branch=0_35_dmabuf_fixes#1fd43557d3acb3ddd2721f61b6939ab636ecd17a"
 dependencies = [
  "proc-macro-crate 3.3.0",
  "proc-macro2",

Cargo.toml

@@ -142,7 +142,7 @@ tokio-stream = { version = "0.1.17", optional = true }
 memmap2 = { version = "0.9.5", optional = true }
 drm-fourcc = { version = "2.2.0", optional = true }
 memfd = { version = "0.6.4", optional = true }
-vulkano = { version = "0.35.1", optional = true }
+vulkano = { git = "https://github.com/Schmarni-Dev/vulkano", branch = "0_35_dmabuf_fixes", optional = true }
 wgpu-hal = { version = "24", optional = true, features = ["vulkan"] }
 ash = { version = "0.38.0", optional = true, default-features = false }
 

src/wayland/core/buffer.rs

@@ -64,7 +64,7 @@ impl Buffer {
 	) -> Option<Handle<Image>> {
 		tracing::debug!("Updating texture for buffer {:?}", self.id);
 		match &self.backing {
-			BufferBacking::Shm(backing) => backing.update_tex(images),
+			BufferBacking::Shm(backing) => backing.update_tex(dmatexes, images),
 			BufferBacking::Dmabuf(backing) => backing.update_tex(dmatexes, images),
 		}
 	}

src/wayland/core/shm_buffer_backing.rs

@@ -1,23 +1,47 @@
+use crate::wayland::{RENDER_DEVICE, vulkano_data::VULKANO_CONTEXT};
+
 use super::shm_pool::ShmPool;
 use bevy::{
-	asset::{Assets, Handle, RenderAssetUsages},
-	image::Image,
-	render::render_resource::{Extent3d, TextureDimension, TextureFormat},
+	asset::{Assets, Handle},
+	image::Image as BevyImage,
+};
+use bevy_dmabuf::{
+	dmatex::{Dmatex, DmatexPlane, Resolution},
+	format_mapping::vk_format_to_drm_fourcc,
+	import::{DropCallback, ImportedDmatexs, ImportedTexture, import_texture},
 };
 use mint::Vector2;
 use parking_lot::Mutex;
-use std::sync::Arc;
+use std::{
+	os::fd::OwnedFd,
+	sync::{Arc, OnceLock},
+};
+use tracing::debug_span;
+use vulkano::{
+	buffer::{BufferCreateFlags, BufferCreateInfo, BufferUsage},
+	command_buffer::{AutoCommandBufferBuilder, CommandBufferUsage, CopyBufferToImageInfo},
+	image::{
+		Image, ImageAspect, ImageCreateFlags, ImageCreateInfo, ImageLayout, ImageMemory,
+		ImageTiling, ImageUsage, sys::RawImage,
+	},
+	memory::{
+		DedicatedAllocation, DeviceMemory, ExternalMemoryHandleType, ExternalMemoryHandleTypes,
+		MemoryAllocateInfo, ResourceMemory, allocator::AllocationCreateInfo,
+	},
+	sync::{self, GpuFuture, Sharing},
+};
 use waynest::server::protocol::core::wayland::wl_shm::Format;
 
 /// Parameters for a shared memory buffer
-#[derive(Debug)]
 pub struct ShmBufferBacking {
 	pool: Arc<ShmPool>,
 	offset: usize,
 	stride: usize,
 	size: Vector2<usize>,
 	format: Format,
-	image: Mutex<Handle<Image>>,
+	image: Arc<Image>,
+	image_handle: OnceLock<Handle<BevyImage>>,
+	pending_imported_dmatex: Mutex<Option<ImportedTexture>>,
 }
 
 impl ShmBufferBacking {
@@ -28,31 +52,137 @@ impl ShmBufferBacking {
 		size: Vector2<usize>,
 		format: Format,
 	) -> Self {
+		// TODO: this might cause a freeze?
+		let vk = VULKANO_CONTEXT.wait();
+		let bevy_render_dev = RENDER_DEVICE.wait();
+
+		let vk_format = vulkano::format::Format::R8G8B8A8_UNORM;
+
+		let modifiers = vk
+			.phys_dev
+			.format_properties(vk_format)
+			.unwrap()
+			.drm_format_modifier_properties
+			.into_iter()
+			.map(|v| v.drm_format_modifier)
+			.collect::<Vec<_>>();
+
+		let raw_image = RawImage::new(
+			vk.dev.clone(),
+			ImageCreateInfo {
+				flags: ImageCreateFlags::empty(),
+				image_type: vulkano::image::ImageType::Dim2d,
+				format: vk_format,
+				view_formats: Vec::new(),
+				extent: [size.x as u32, size.y as u32, 1],
+				tiling: ImageTiling::DrmFormatModifier,
+				usage: ImageUsage::COLOR_ATTACHMENT
+					| ImageUsage::SAMPLED
+					| ImageUsage::TRANSFER_DST,
+				initial_layout: ImageLayout::Undefined,
+				drm_format_modifiers: modifiers,
+				external_memory_handle_types: ExternalMemoryHandleTypes::DMA_BUF,
+				..Default::default()
+			},
+		)
+		.unwrap();
+		let (modifier, num_planes) = raw_image.drm_format_modifier().unwrap();
+		let mem_reqs = raw_image.memory_requirements()[0];
+		let index = vk
+			.phys_dev
+			.memory_properties()
+			.memory_types
+			.iter()
+			.enumerate()
+			.map(|(i, _v)| i as u32)
+			.find(|i| mem_reqs.memory_type_bits & (1 << i) != 0)
+			.expect("no valid memory type");
+		let mem = ResourceMemory::new_dedicated(
+			DeviceMemory::allocate(
+				vk.dev.clone(),
+				MemoryAllocateInfo {
+					allocation_size: mem_reqs.layout.size(),
+					memory_type_index: index,
+					dedicated_allocation: Some(DedicatedAllocation::Image(&raw_image)),
+					export_handle_types: ExternalMemoryHandleTypes::DMA_BUF,
+					..Default::default()
+				},
+			)
+			.unwrap(),
+		);
+
+		let image = Arc::new(match raw_image.bind_memory([mem]) {
+			Ok(v) => v,
+			Err(_) => panic!("unable to bind memory"),
+		});
+		let ImageMemory::Normal(mem) = image.memory() else {
+			unreachable!()
+		};
+		let [mem] = mem.as_slice() else {
+			unreachable!()
+		};
+		let fd = OwnedFd::from(
+			mem.device_memory()
+				.export_fd(ExternalMemoryHandleType::DmaBuf)
+				.unwrap(),
+		);
+		let planes = (0..num_planes)
+			.filter_map(|i| {
+				Some(match i {
+					0 => ImageAspect::MemoryPlane0,
+					1 => ImageAspect::MemoryPlane1,
+					2 => ImageAspect::MemoryPlane2,
+					3 => ImageAspect::MemoryPlane3,
+					_ => return None,
+				})
+			})
+			.map(|aspect| {
+				let plane_layout = image.subresource_layout(aspect, 0, 0).unwrap();
+
+				DmatexPlane {
+					dmabuf_fd: fd.try_clone().unwrap().into(),
+					modifier,
+					offset: plane_layout.offset as u32,
+					stride: plane_layout.row_pitch as i32,
+				}
+			})
+			.collect::<Vec<_>>();
+		let dmatex = Dmatex {
+			planes,
+			res: Resolution {
+				x: size.x as u32,
+				y: size.y as u32,
+			},
+			format: vk_format_to_drm_fourcc(vk_format.into()).unwrap() as u32,
+			flip_y: false,
+		};
+
+		let imported_texture = import_texture(bevy_render_dev, dmatex, DropCallback(None)).unwrap();
 		Self {
 			pool,
 			offset,
 			stride,
 			size,
 			format,
-			image: Mutex::new(Handle::default()),
+			image,
+			image_handle: OnceLock::new(),
+			pending_imported_dmatex: Mutex::new(Some(imported_texture)),
 		}
 	}
 
 	#[tracing::instrument("debug", skip_all)]
-	pub fn update_tex(&self, images: &mut Assets<Image>) -> Option<Handle<Image>> {
-		let mut image_handle = self.image.lock();
-		images.remove(image_handle.id());
-		let mut image = Image::new_fill(
-			Extent3d {
-				width: self.size.x as u32,
-				height: self.size.y as u32,
-				depth_or_array_layers: 1,
-			},
-			TextureDimension::D2,
-			&[255, 0, 255, 255],
-			TextureFormat::Rgba8UnormSrgb,
-			RenderAssetUsages::all(),
-		);
+	pub fn update_tex(
+		&self,
+		dmatexes: &ImportedDmatexs,
+		images: &mut Assets<BevyImage>,
+	) -> Option<Handle<BevyImage>> {
+		if let Some(tex) = self.pending_imported_dmatex.lock().take() {
+			self.image_handle
+				.set(dmatexes.insert_imported_dmatex(images, tex))
+				.unwrap();
+		}
+		let vk = VULKANO_CONTEXT.wait();
+		let image_handle = self.image_handle.get().unwrap();
 
 		let src_data_lock = self.pool.data_lock();
 		let mut src_cursor = self.offset;
@@ -64,13 +194,26 @@ impl ShmBufferBacking {
 		if max_cursor > src_data_lock.len() {
 			return None;
 		}
-
-		let dst_data = image.data.get_or_insert_with(|| {
-			let length = self.size.x * self.size.y * 4;
-			vec![255; length]
-		});
+		let data_len = (self.size.x * self.size.y * 4) as u64;
 		let mut dst_cursor = 0;
 
+		let buffer = vulkano::buffer::Buffer::new_slice::<u8>(
+			vk.alloc.clone(),
+			BufferCreateInfo {
+				flags: BufferCreateFlags::empty(),
+				sharing: Sharing::Exclusive,
+				usage: BufferUsage::TRANSFER_SRC,
+				..Default::default()
+			},
+			AllocationCreateInfo {
+				memory_type_filter:
+					vulkano::memory::allocator::MemoryTypeFilter::HOST_SEQUENTIAL_WRITE,
+				..Default::default()
+			},
+			data_len,
+		)
+		.unwrap();
+		let mut dst_data = buffer.write().unwrap();
 		for _y in 0..self.size.y {
 			for _x in 0..self.size.x {
 				match self.format {
@@ -87,14 +230,37 @@ impl ShmBufferBacking {
 			}
 			src_cursor += self.stride - (self.size.x * 4);
 		}
+		drop(dst_data);
+
+		let mut command_buffer = AutoCommandBufferBuilder::primary(
+			vk.command_buffer_alloc.clone(),
+			vk.queue.queue_family_index(),
+			CommandBufferUsage::OneTimeSubmit,
+		)
+		.unwrap();
+		command_buffer
+			.copy_buffer_to_image(CopyBufferToImageInfo::buffer_image(
+				buffer.clone(),
+				self.image.clone(),
+			))
+			.unwrap();
+		let command_buffer = command_buffer.build().unwrap();
+		debug_span!("waiting for buffer copy").in_scope(|| {
+			sync::now(vk.dev.clone())
+				.then_execute(vk.queue.clone(), command_buffer)
+				.unwrap()
+				.then_signal_fence_and_flush()
+				.unwrap()
+				.wait(None)
+				.unwrap()
+		});
 
-		*image_handle = images.add(image);
 		Some(image_handle.clone())
 	}
 
 	pub fn is_transparent(&self) -> bool {
 		match self.format {
-			Format::Xrgb8888 => true,
+			Format::Xrgb8888 => false,
 			Format::Argb8888 => true,
 			_ => true,
 		}
@@ -104,3 +270,17 @@ impl ShmBufferBacking {
 		self.size
 	}
 }
+
+impl std::fmt::Debug for ShmBufferBacking {
+	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+		f.debug_struct("ShmBufferBacking")
+			.field("pool", &self.pool)
+			.field("offset", &self.offset)
+			.field("stride", &self.stride)
+			.field("size", &self.size)
+			.field("format", &self.format)
+			.field("image", &self.image)
+			.field("image_handle", &self.image_handle)
+			.finish()
+	}
+}

src/wayland/vulkano_data.rs

@@ -50,7 +50,9 @@ pub fn setup_vulkano_context(
 			ash_instance.handle(),
 			vulkano::instance::InstanceCreateInfo {
 				flags: InstanceCreateFlags::empty(),
-				max_api_version: Some(vulkano::Version::from(hal_instance.instance_api_version())),
+				// TODO: make vulkan init reasonable and remove this hardcoded value from
+				// bevy_mod_openxr
+				max_api_version: Some(vulkano::Version::V1_2),
 				enabled_extensions: vulkano::instance::InstanceExtensions::from_iter(
 					hal_instance
 						.extensions()
@@ -93,6 +95,7 @@ pub fn setup_vulkano_context(
 					),
 					// this is def wrong, lets hope it doesn't cause issues....
 					enabled_features: vulkano::device::DeviceFeatures::empty(),
+
 					..Default::default()
 				},
 			)