servius/jello
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: Try to minimize frame latency

Browse Source
This commit is contained in:
uttarayan21
2025-12-25 05:48:51 +05:30
parent 5d0b795ba5
commit 5a0bdae84b
7 changed files with 72 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
crates/iced-video/Cargo.toml
View File

@@ -16,3 +16,8 @@ tracing = "0.1.43"
[dev-dependencies]
iced.workspace = true
tracing-subscriber = { version = "0.3.22", features = ["env-filter"] }
[profile.dev]
debug = true
[profile.release]
debug = true

61
crates/iced-video/src/primitive.rs
View File

@@ -9,7 +9,7 @@ pub struct VideoFrame {
pub id: id::Id,
pub size: wgpu::Extent3d,
pub ready: Arc<AtomicBool>,
pub frame: Arc<Mutex<Vec<u8>>>,
pub frame: Arc<Mutex<gst::app::Sample>>,
}
impl iced_wgpu::Primitive for VideoFrame {
@@ -34,6 +34,12 @@ impl iced_wgpu::Primitive for VideoFrame {
usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
view_formats: &[],
});
let buffer = device.create_buffer(&wgpu::BufferDescriptor {
label: Some("iced-video-buffer"),
size: (self.size.width * self.size.height * 4) as u64,
usage: wgpu::BufferUsages::COPY_SRC | wgpu::BufferUsages::COPY_DST,
mapped_at_creation: false,
});
let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor {
label: Some("iced-video-texture-bind-group"),
layout: &pipeline.bind_group_layout,
@@ -53,6 +59,7 @@ impl iced_wgpu::Primitive for VideoFrame {
VideoTextures {
id: self.id.clone(),
texture,
buffer,
bind_group,
ready: Arc::clone(&self.ready),
}
@@ -89,35 +96,18 @@ impl iced_wgpu::Primitive for VideoFrame {
video.texture = new_texture;
video.bind_group = new_bind_group;
}
// BUG: This causes a panic because the texture size is not correct for some reason.
if video.ready.load(std::sync::atomic::Ordering::SeqCst) {
let now = std::time::Instant::now();
let frame = self.frame.lock().expect("BUG: Mutex poisoned");
if frame.len() != (4 * self.size.width * self.size.height) as usize {
tracing::warn!(
"Frame size mismatch: expected {}, got {}",
4 * self.size.width * self.size.height,
frame.len()
);
return;
}
queue.write_texture(
wgpu::TexelCopyTextureInfo {
texture: &video.texture,
mip_level: 0,
origin: wgpu::Origin3d::ZERO,
aspect: wgpu::TextureAspect::All,
},
&frame,
wgpu::TexelCopyBufferLayout {
offset: 0,
bytes_per_row: Some(4 * video.texture.size().width),
rows_per_image: Some(video.texture.size().height),
},
self.size,
);
let frame = frame
.buffer()
.and_then(|b| b.map_readable().ok())
.expect("BUG: Failed to get frame data from gst::Sample");
queue.write_buffer(&video.buffer, 0, &frame);
video
.ready
.store(false, std::sync::atomic::Ordering::SeqCst);
tracing::info!("{:?} Taken to write to surface texture", now.elapsed());
}
}
@@ -126,11 +116,29 @@ impl iced_wgpu::Primitive for VideoFrame {
pipeline: &Self::Pipeline,
encoder: &mut wgpu::CommandEncoder,
target: &wgpu::TextureView,
clip_bounds: &iced_wgpu::core::Rectangle<u32>,
_clip_bounds: &iced_wgpu::core::Rectangle<u32>,
) {
let Some(video) = pipeline.videos.get(&self.id) else {
return;
};
encoder.copy_buffer_to_texture(
wgpu::TexelCopyBufferInfo {
buffer: &video.buffer,
layout: wgpu::TexelCopyBufferLayout {
offset: 0,
bytes_per_row: Some(4 * self.size.width),
rows_per_image: Some(self.size.height),
},
},
wgpu::TexelCopyTextureInfo {
texture: &video.texture,
mip_level: 0,
origin: wgpu::Origin3d::ZERO,
aspect: wgpu::TextureAspect::All,
},
self.size,
);
let mut render_pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor {
label: Some("iced-video-render-pass"),
color_attachments: &[Some(wgpu::RenderPassColorAttachment {
@@ -164,6 +172,7 @@ impl iced_wgpu::Primitive for VideoFrame {
pub struct VideoTextures {
id: id::Id,
texture: wgpu::Texture,
buffer: wgpu::Buffer,
bind_group: wgpu::BindGroup,
ready: Arc<AtomicBool>,
}

0
crates/iced-video/src/shaders/bt.2020.wgsl
View File

0
crates/iced-video/src/shaders/bt.709.wgsl
View File

39
crates/iced-video/src/source.rs
View File

@@ -17,7 +17,7 @@ pub struct VideoSource {
pub(crate) appsink: AppSink,
pub(crate) bus: Bus,
pub(crate) ready: Arc<AtomicBool>,
pub(crate) frame: Arc<Mutex<Vec<u8>>>,
pub(crate) frame: Arc<Mutex<gst::app::Sample>>,
}
impl VideoSource {
@@ -45,41 +45,26 @@ impl VideoSource {
let bus = playbin.bus().change_context(Error)?;
playbin.pause().change_context(Error)?;
let ready = Arc::new(AtomicBool::new(false));
let frame = Arc::new(Mutex::new(Vec::new()));
let frame = Arc::new(Mutex::new(gst::app::Sample::new()));
let appsink = appsink.on_new_frame({
let ready = Arc::clone(&ready);
let frame = Arc::clone(&frame);
move |appsink| {
let Ok(sample) = appsink.pull_sample() else {
tracing::error!("Failed to pull video sample from appsink despite being notified of new frame");
return Ok(());
};
let caps = sample.caps().ok_or(gst::gstreamer::FlowError::Error)?;
let structure_0 = caps.structure(0).ok_or(gst::gstreamer::FlowError::Error)?;
let width = structure_0
.get::<i32>("width")
.map_err(|_| gst::gstreamer::FlowError::Error)?;
let height = structure_0
.get::<i32>("height")
.map_err(|_| gst::gstreamer::FlowError::Error)?;
let buffer = sample.buffer().and_then(|b| b.map_readable().ok());
if let Some(buffer) = buffer {
{
let mut frame = frame.lock().expect("BUG: Mutex poisoned");
debug_assert_eq!(buffer.size(), (width * height * 4) as usize);
if frame.len() != buffer.size() {
frame.resize(buffer.size(), 0);
}
frame.copy_from_slice(buffer.as_slice());
ready.store(true, std::sync::atomic::Ordering::Relaxed);
}
// if written.is_err() {
// tracing::error!("Failed to write video frame to buffer");
// } else {
// ready.store(true, std::sync::atomic::Ordering::Relaxed);
// }
{
let mut guard = frame.lock().expect("BUG: Mutex poisoned");
let old_sample = core::mem::replace(&mut *guard, sample);
// dbg!(old_sample.caps());
// dbg!(old_sample.inner.buffer_list());
// drop(old_sample);
ready.store(true, std::sync::atomic::Ordering::Relaxed);
drop(guard);
}
Ok(())
}
});

3
flake.nix
View File

@@ -201,6 +201,9 @@
apple-sdk_26
])
++ (lib.optionals pkgs.stdenv.isLinux [
valgrind
hotspot
samply
cargo-flamegraph
perf
mold

18
gst/src/plugins/app/appsink.rs
View File

@@ -111,12 +111,20 @@ impl From<gstreamer::Sample> for Sample {
}
#[repr(transparent)]
#[derive(Debug, Clone)]
pub struct Sample {
inner: gstreamer::Sample,
pub inner: gstreamer::Sample,
}
use gstreamer::BufferRef;
impl Sample {
#[doc(alias = "empty")]
pub fn new() -> Self {
Self {
inner: gstreamer::Sample::builder().build(),
}
}
pub fn buffer(&self) -> Option<&BufferRef> {
self.inner.buffer()
}
@@ -124,6 +132,14 @@ impl Sample {
pub fn caps(&self) -> Option<&gstreamer::CapsRef> {
self.inner.caps()
}
pub fn info(&self) -> Option<&gstreamer::StructureRef> {
self.inner.info()
}
// pub fn set_buffer(&mut self) {
// self.inner.set_buffer(None);
// }
}
#[test]