fix: Try to minimize frame latency

crates/iced-video/src/primitive.rs

@@ -9,7 +9,7 @@ pub struct VideoFrame {
     pub id: id::Id,
     pub size: wgpu::Extent3d,
     pub ready: Arc<AtomicBool>,
-    pub frame: Arc<Mutex<Vec<u8>>>,
+    pub frame: Arc<Mutex<gst::app::Sample>>,
 }
 
 impl iced_wgpu::Primitive for VideoFrame {
@@ -34,6 +34,12 @@ impl iced_wgpu::Primitive for VideoFrame {
                 usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
                 view_formats: &[],
             });
+            let buffer = device.create_buffer(&wgpu::BufferDescriptor {
+                label: Some("iced-video-buffer"),
+                size: (self.size.width * self.size.height * 4) as u64,
+                usage: wgpu::BufferUsages::COPY_SRC | wgpu::BufferUsages::COPY_DST,
+                mapped_at_creation: false,
+            });
             let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor {
                 label: Some("iced-video-texture-bind-group"),
                 layout: &pipeline.bind_group_layout,
@@ -53,6 +59,7 @@ impl iced_wgpu::Primitive for VideoFrame {
             VideoTextures {
                 id: self.id.clone(),
                 texture,
+                buffer,
                 bind_group,
                 ready: Arc::clone(&self.ready),
             }
@@ -89,35 +96,18 @@ impl iced_wgpu::Primitive for VideoFrame {
             video.texture = new_texture;
             video.bind_group = new_bind_group;
         }
-        // BUG: This causes a panic because the texture size is not correct for some reason.
         if video.ready.load(std::sync::atomic::Ordering::SeqCst) {
+            let now = std::time::Instant::now();
             let frame = self.frame.lock().expect("BUG: Mutex poisoned");
-            if frame.len() != (4 * self.size.width * self.size.height) as usize {
-                tracing::warn!(
-                    "Frame size mismatch: expected {}, got {}",
-                    4 * self.size.width * self.size.height,
-                    frame.len()
-                );
-                return;
-            }
-            queue.write_texture(
-                wgpu::TexelCopyTextureInfo {
-                    texture: &video.texture,
-                    mip_level: 0,
-                    origin: wgpu::Origin3d::ZERO,
-                    aspect: wgpu::TextureAspect::All,
-                },
-                &frame,
-                wgpu::TexelCopyBufferLayout {
-                    offset: 0,
-                    bytes_per_row: Some(4 * video.texture.size().width),
-                    rows_per_image: Some(video.texture.size().height),
-                },
-                self.size,
-            );
+            let frame = frame
+                .buffer()
+                .and_then(|b| b.map_readable().ok())
+                .expect("BUG: Failed to get frame data from gst::Sample");
+            queue.write_buffer(&video.buffer, 0, &frame);
             video
                 .ready
                 .store(false, std::sync::atomic::Ordering::SeqCst);
+            tracing::info!("{:?} Taken to write to surface texture", now.elapsed());
         }
     }
 
@@ -126,11 +116,29 @@ impl iced_wgpu::Primitive for VideoFrame {
         pipeline: &Self::Pipeline,
         encoder: &mut wgpu::CommandEncoder,
         target: &wgpu::TextureView,
-        clip_bounds: &iced_wgpu::core::Rectangle<u32>,
+        _clip_bounds: &iced_wgpu::core::Rectangle<u32>,
     ) {
         let Some(video) = pipeline.videos.get(&self.id) else {
             return;
         };
+
+        encoder.copy_buffer_to_texture(
+            wgpu::TexelCopyBufferInfo {
+                buffer: &video.buffer,
+                layout: wgpu::TexelCopyBufferLayout {
+                    offset: 0,
+                    bytes_per_row: Some(4 * self.size.width),
+                    rows_per_image: Some(self.size.height),
+                },
+            },
+            wgpu::TexelCopyTextureInfo {
+                texture: &video.texture,
+                mip_level: 0,
+                origin: wgpu::Origin3d::ZERO,
+                aspect: wgpu::TextureAspect::All,
+            },
+            self.size,
+        );
         let mut render_pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor {
             label: Some("iced-video-render-pass"),
             color_attachments: &[Some(wgpu::RenderPassColorAttachment {
@@ -164,6 +172,7 @@ impl iced_wgpu::Primitive for VideoFrame {
 pub struct VideoTextures {
     id: id::Id,
     texture: wgpu::Texture,
+    buffer: wgpu::Buffer,
     bind_group: wgpu::BindGroup,
     ready: Arc<AtomicBool>,
 }