From 5a0bdae84be4f20fd1f1b64e6fc58e6591108d26 Mon Sep 17 00:00:00 2001 From: uttarayan21 Date: Thu, 25 Dec 2025 05:48:51 +0530 Subject: [PATCH] fix: Try to minimize frame latency --- crates/iced-video/Cargo.toml | 5 ++ crates/iced-video/src/primitive.rs | 61 +++++++++++++--------- crates/iced-video/src/shaders/bt.2020.wgsl | 0 crates/iced-video/src/shaders/bt.709.wgsl | 0 crates/iced-video/src/source.rs | 39 +++++--------- flake.nix | 3 ++ gst/src/plugins/app/appsink.rs | 18 ++++++- 7 files changed, 72 insertions(+), 54 deletions(-) delete mode 100644 crates/iced-video/src/shaders/bt.2020.wgsl delete mode 100644 crates/iced-video/src/shaders/bt.709.wgsl diff --git a/crates/iced-video/Cargo.toml b/crates/iced-video/Cargo.toml index 0b9ba3a..126c8a4 100644 --- a/crates/iced-video/Cargo.toml +++ b/crates/iced-video/Cargo.toml @@ -16,3 +16,8 @@ tracing = "0.1.43" [dev-dependencies] iced.workspace = true tracing-subscriber = { version = "0.3.22", features = ["env-filter"] } + +[profile.dev] +debug = true +[profile.release] +debug = true diff --git a/crates/iced-video/src/primitive.rs b/crates/iced-video/src/primitive.rs index b0469ed..d57c2e8 100644 --- a/crates/iced-video/src/primitive.rs +++ b/crates/iced-video/src/primitive.rs @@ -9,7 +9,7 @@ pub struct VideoFrame { pub id: id::Id, pub size: wgpu::Extent3d, pub ready: Arc, - pub frame: Arc>>, + pub frame: Arc>, } impl iced_wgpu::Primitive for VideoFrame { @@ -34,6 +34,12 @@ impl iced_wgpu::Primitive for VideoFrame { usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, view_formats: &[], }); + let buffer = device.create_buffer(&wgpu::BufferDescriptor { + label: Some("iced-video-buffer"), + size: (self.size.width * self.size.height * 4) as u64, + usage: wgpu::BufferUsages::COPY_SRC | wgpu::BufferUsages::COPY_DST, + mapped_at_creation: false, + }); let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor { label: Some("iced-video-texture-bind-group"), layout: &pipeline.bind_group_layout, @@ -53,6 +59,7 @@ impl iced_wgpu::Primitive for VideoFrame { VideoTextures { id: self.id.clone(), texture, + buffer, bind_group, ready: Arc::clone(&self.ready), } @@ -89,35 +96,18 @@ impl iced_wgpu::Primitive for VideoFrame { video.texture = new_texture; video.bind_group = new_bind_group; } - // BUG: This causes a panic because the texture size is not correct for some reason. if video.ready.load(std::sync::atomic::Ordering::SeqCst) { + let now = std::time::Instant::now(); let frame = self.frame.lock().expect("BUG: Mutex poisoned"); - if frame.len() != (4 * self.size.width * self.size.height) as usize { - tracing::warn!( - "Frame size mismatch: expected {}, got {}", - 4 * self.size.width * self.size.height, - frame.len() - ); - return; - } - queue.write_texture( - wgpu::TexelCopyTextureInfo { - texture: &video.texture, - mip_level: 0, - origin: wgpu::Origin3d::ZERO, - aspect: wgpu::TextureAspect::All, - }, - &frame, - wgpu::TexelCopyBufferLayout { - offset: 0, - bytes_per_row: Some(4 * video.texture.size().width), - rows_per_image: Some(video.texture.size().height), - }, - self.size, - ); + let frame = frame + .buffer() + .and_then(|b| b.map_readable().ok()) + .expect("BUG: Failed to get frame data from gst::Sample"); + queue.write_buffer(&video.buffer, 0, &frame); video .ready .store(false, std::sync::atomic::Ordering::SeqCst); + tracing::info!("{:?} Taken to write to surface texture", now.elapsed()); } } @@ -126,11 +116,29 @@ impl iced_wgpu::Primitive for VideoFrame { pipeline: &Self::Pipeline, encoder: &mut wgpu::CommandEncoder, target: &wgpu::TextureView, - clip_bounds: &iced_wgpu::core::Rectangle, + _clip_bounds: &iced_wgpu::core::Rectangle, ) { let Some(video) = pipeline.videos.get(&self.id) else { return; }; + + encoder.copy_buffer_to_texture( + wgpu::TexelCopyBufferInfo { + buffer: &video.buffer, + layout: wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(4 * self.size.width), + rows_per_image: Some(self.size.height), + }, + }, + wgpu::TexelCopyTextureInfo { + texture: &video.texture, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + self.size, + ); let mut render_pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { label: Some("iced-video-render-pass"), color_attachments: &[Some(wgpu::RenderPassColorAttachment { @@ -164,6 +172,7 @@ impl iced_wgpu::Primitive for VideoFrame { pub struct VideoTextures { id: id::Id, texture: wgpu::Texture, + buffer: wgpu::Buffer, bind_group: wgpu::BindGroup, ready: Arc, } diff --git a/crates/iced-video/src/shaders/bt.2020.wgsl b/crates/iced-video/src/shaders/bt.2020.wgsl deleted file mode 100644 index e69de29..0000000 diff --git a/crates/iced-video/src/shaders/bt.709.wgsl b/crates/iced-video/src/shaders/bt.709.wgsl deleted file mode 100644 index e69de29..0000000 diff --git a/crates/iced-video/src/source.rs b/crates/iced-video/src/source.rs index b60d702..437f8aa 100644 --- a/crates/iced-video/src/source.rs +++ b/crates/iced-video/src/source.rs @@ -17,7 +17,7 @@ pub struct VideoSource { pub(crate) appsink: AppSink, pub(crate) bus: Bus, pub(crate) ready: Arc, - pub(crate) frame: Arc>>, + pub(crate) frame: Arc>, } impl VideoSource { @@ -45,41 +45,26 @@ impl VideoSource { let bus = playbin.bus().change_context(Error)?; playbin.pause().change_context(Error)?; let ready = Arc::new(AtomicBool::new(false)); - let frame = Arc::new(Mutex::new(Vec::new())); + let frame = Arc::new(Mutex::new(gst::app::Sample::new())); let appsink = appsink.on_new_frame({ let ready = Arc::clone(&ready); let frame = Arc::clone(&frame); move |appsink| { let Ok(sample) = appsink.pull_sample() else { + tracing::error!("Failed to pull video sample from appsink despite being notified of new frame"); return Ok(()); }; - let caps = sample.caps().ok_or(gst::gstreamer::FlowError::Error)?; - let structure_0 = caps.structure(0).ok_or(gst::gstreamer::FlowError::Error)?; - let width = structure_0 - .get::("width") - .map_err(|_| gst::gstreamer::FlowError::Error)?; - let height = structure_0 - .get::("height") - .map_err(|_| gst::gstreamer::FlowError::Error)?; - - let buffer = sample.buffer().and_then(|b| b.map_readable().ok()); - if let Some(buffer) = buffer { - { - let mut frame = frame.lock().expect("BUG: Mutex poisoned"); - debug_assert_eq!(buffer.size(), (width * height * 4) as usize); - if frame.len() != buffer.size() { - frame.resize(buffer.size(), 0); - } - frame.copy_from_slice(buffer.as_slice()); - ready.store(true, std::sync::atomic::Ordering::Relaxed); - } - // if written.is_err() { - // tracing::error!("Failed to write video frame to buffer"); - // } else { - // ready.store(true, std::sync::atomic::Ordering::Relaxed); - // } + { + let mut guard = frame.lock().expect("BUG: Mutex poisoned"); + let old_sample = core::mem::replace(&mut *guard, sample); + // dbg!(old_sample.caps()); + // dbg!(old_sample.inner.buffer_list()); + // drop(old_sample); + ready.store(true, std::sync::atomic::Ordering::Relaxed); + drop(guard); } + Ok(()) } }); diff --git a/flake.nix b/flake.nix index 96de60f..30f50df 100644 --- a/flake.nix +++ b/flake.nix @@ -201,6 +201,9 @@ apple-sdk_26 ]) ++ (lib.optionals pkgs.stdenv.isLinux [ + valgrind + hotspot + samply cargo-flamegraph perf mold diff --git a/gst/src/plugins/app/appsink.rs b/gst/src/plugins/app/appsink.rs index b1b17fd..f9427c5 100644 --- a/gst/src/plugins/app/appsink.rs +++ b/gst/src/plugins/app/appsink.rs @@ -111,12 +111,20 @@ impl From for Sample { } #[repr(transparent)] +#[derive(Debug, Clone)] pub struct Sample { - inner: gstreamer::Sample, + pub inner: gstreamer::Sample, } use gstreamer::BufferRef; impl Sample { + #[doc(alias = "empty")] + pub fn new() -> Self { + Self { + inner: gstreamer::Sample::builder().build(), + } + } + pub fn buffer(&self) -> Option<&BufferRef> { self.inner.buffer() } @@ -124,6 +132,14 @@ impl Sample { pub fn caps(&self) -> Option<&gstreamer::CapsRef> { self.inner.caps() } + + pub fn info(&self) -> Option<&gstreamer::StructureRef> { + self.inner.info() + } + + // pub fn set_buffer(&mut self) { + // self.inner.set_buffer(None); + // } } #[test]