feat(iced-video): implement planar YUV texture support with HDR conversion matrices and update dependencies

crates/iced-video/Cargo.toml

@@ -4,6 +4,7 @@ version = "0.1.0"
 edition = "2024"
 
 [dependencies]
+bytemuck = "1.24.0"
 error-stack = "0.6.0"
 futures-lite = "2.6.1"
 gst.workspace = true
@@ -13,6 +14,7 @@ iced_renderer = { version = "0.14.0", features = ["iced_wgpu"] }
 iced_wgpu = { version = "0.14.0" }
 thiserror = "2.0.17"
 tracing = "0.1.43"
+wgpu = { version = "27.0.1", features = ["vulkan"] }
 
 [dev-dependencies]
 iced.workspace = true

crates/iced-video/src/primitive.rs

@@ -1,9 +1,65 @@
 use crate::id;
+use gst::videoconvertscale::VideoFormat;
 use iced_wgpu::primitive::Pipeline;
 use iced_wgpu::wgpu;
 use std::collections::BTreeMap;
 use std::sync::{Arc, Mutex, atomic::AtomicBool};
 
+#[derive(Clone, Copy, Debug, bytemuck::Zeroable, bytemuck::Pod)]
+#[repr(transparent)]
+pub struct ConversionMatrix {
+    matrix: [[f32; 4]; 4],
+}
+
+// impl ConversionMatrix {
+//     pub fn desc() -> wgpu::VertexBufferLayout<'static> {
+//         wgpu::VertexBufferLayout {
+//             array_stride: core::mem::size_of::<ConversionMatrix>() as wgpu::BufferAddress,
+//             step_mode: wgpu::VertexStepMode::Vertex,
+//             attributes: &[
+//                 wgpu::VertexAttribute {
+//                     offset: 0,
+//                     shader_location: 0,
+//                     format: wgpu::VertexFormat::Float32x4,
+//                 },
+//                 wgpu::VertexAttribute {
+//                     offset: 16,
+//                     shader_location: 1,
+//                     format: wgpu::VertexFormat::Float32x4,
+//                 },
+//                 wgpu::VertexAttribute {
+//                     offset: 32,
+//                     shader_location: 2,
+//                     format: wgpu::VertexFormat::Float32x4,
+//                 },
+//                 wgpu::VertexAttribute {
+//                     offset: 48,
+//                     shader_location: 3,
+//                     format: wgpu::VertexFormat::Float32x4,
+//                 },
+//             ],
+//         }
+//     }
+// }
+
+pub const BT2020_TO_RGB: ConversionMatrix = ConversionMatrix {
+    matrix: [
+        [1.1684, 0.0000, 1.6836, -0.9122],
+        [1.1684, -0.1873, -0.6520, 0.3015],
+        [1.1684, 2.1482, 0.0000, -1.1322],
+        [0.0, 0.0, 0.0, 1.0],
+    ],
+};
+
+pub const BT709_TO_RGB: ConversionMatrix = ConversionMatrix {
+    matrix: [
+        [1.1644, 0.0000, 1.7927, -0.9729],
+        [1.1644, -0.2132, -0.5329, 0.3015],
+        [1.1644, 2.1124, 0.0000, -1.1334],
+        [0.0, 0.0, 0.0, 1.0],
+    ],
+};
+
 #[derive(Debug)]
 pub struct VideoFrame {
     pub id: id::Id,
@@ -24,73 +80,78 @@ impl iced_wgpu::Primitive for VideoFrame {
         viewport: &iced_wgpu::graphics::Viewport,
     ) {
         let video = pipeline.videos.entry(self.id.clone()).or_insert_with(|| {
-            let texture = device.create_texture(&wgpu::TextureDescriptor {
-                label: Some("iced-video-texture"),
-                size: self.size,
-                mip_level_count: 1,
-                sample_count: 1,
-                dimension: wgpu::TextureDimension::D2,
-                format: pipeline.format,
-                usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
-                view_formats: &[],
-            });
+            let texture =
+                VideoTexture::new("iced-video-texture", self.size, device, pipeline.format);
+            let conversion_matrix = if texture.format().is_wide() {
+                BT2020_TO_RGB
+            } else {
+                BT709_TO_RGB
+            };
             let buffer = device.create_buffer(&wgpu::BufferDescriptor {
-                label: Some("iced-video-buffer"),
-                size: (self.size.width * self.size.height * 4) as u64,
-                usage: wgpu::BufferUsages::COPY_SRC | wgpu::BufferUsages::COPY_DST,
+                label: Some("iced-video-conversion-matrix-buffer"),
+                usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST,
+                size: core::mem::size_of::<ConversionMatrix>() as wgpu::BufferAddress,
                 mapped_at_creation: false,
             });
+
             let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor {
                 label: Some("iced-video-texture-bind-group"),
                 layout: &pipeline.bind_group_layout,
                 entries: &[
                     wgpu::BindGroupEntry {
                         binding: 0,
-                        resource: wgpu::BindingResource::TextureView(
-                            &texture.create_view(&wgpu::TextureViewDescriptor::default()),
-                        ),
+                        resource: wgpu::BindingResource::TextureView(&texture.y_texture()),
                     },
                     wgpu::BindGroupEntry {
                         binding: 1,
+                        resource: wgpu::BindingResource::TextureView(&texture.uv_texture()),
+                    },
+                    wgpu::BindGroupEntry {
+                        binding: 2,
                         resource: wgpu::BindingResource::Sampler(&pipeline.sampler),
                     },
+                    wgpu::BindGroupEntry {
+                        binding: 3,
+                        resource: wgpu::BindingResource::Buffer(buffer.as_entire_buffer_binding()),
+                    },
                 ],
             });
-            VideoTextures {
+
+            VideoFrameData {
                 id: self.id.clone(),
                 texture,
-                buffer,
+                conversion_matrix: buffer,
                 bind_group,
                 ready: Arc::clone(&self.ready),
             }
         });
-        // dbg!(&self.size, video.texture.size());
         if self.size != video.texture.size() {
-            // Resize the texture if the size has changed.
-            let new_texture = device.create_texture(&wgpu::TextureDescriptor {
-                label: Some("iced-video-texture-resized"),
-                size: self.size,
-                mip_level_count: 1,
-                sample_count: 1,
-                dimension: wgpu::TextureDimension::D2,
-                format: pipeline.format,
-                usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
-                view_formats: &[],
-            });
+            let new_texture = video
+                .texture
+                .resize("iced-video-texture-resized", self.size, device);
+
             let new_bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor {
-                label: Some("iced-video-texture-bind-group-resized"),
+                label: Some("iced-video-texture-bind-group"),
                 layout: &pipeline.bind_group_layout,
                 entries: &[
                     wgpu::BindGroupEntry {
                         binding: 0,
-                        resource: wgpu::BindingResource::TextureView(
-                            &new_texture.create_view(&wgpu::TextureViewDescriptor::default()),
-                        ),
+                        resource: wgpu::BindingResource::TextureView(&new_texture.y_texture()),
                     },
                     wgpu::BindGroupEntry {
                         binding: 1,
+                        resource: wgpu::BindingResource::TextureView(&new_texture.uv_texture()),
+                    },
+                    wgpu::BindGroupEntry {
+                        binding: 2,
                         resource: wgpu::BindingResource::Sampler(&pipeline.sampler),
                     },
+                    wgpu::BindGroupEntry {
+                        binding: 3,
+                        resource: wgpu::BindingResource::Buffer(
+                            video.conversion_matrix.as_entire_buffer_binding(),
+                        ),
+                    },
                 ],
             });
             video.texture = new_texture;
@@ -106,21 +167,24 @@ impl iced_wgpu::Primitive for VideoFrame {
                 .map_readable()
                 .expect("BUG: Failed to map gst::Buffer readable");
             // queue.write_buffer(&video.buffer, 0, &data);
-            queue.write_texture(
-                wgpu::TexelCopyTextureInfo {
-                    texture: &video.texture,
-                    mip_level: 0,
-                    origin: wgpu::Origin3d::ZERO,
-                    aspect: wgpu::TextureAspect::All,
-                },
-                &data,
-                wgpu::TexelCopyBufferLayout {
-                    offset: 0,
-                    bytes_per_row: Some(4 * self.size.width),
-                    rows_per_image: Some(self.size.height),
-                },
-                self.size,
-            );
+
+            video.texture.write_texture(&data, queue);
+            // queue.write_texture(
+            //     wgpu::TexelCopyTextureInfo {
+            //         texture: &video.texture,
+            //         mip_level: 0,
+            //         origin: wgpu::Origin3d::ZERO,
+            //         aspect: wgpu::TextureAspect::All,
+            //     },
+            //     &data,
+            //     wgpu::TexelCopyBufferLayout {
+            //         offset: 0,
+            //         bytes_per_row: Some(4 * self.size.width),
+            //         rows_per_image: Some(self.size.height),
+            //     },
+            //     self.size,
+            // );
+
             drop(data);
             video
                 .ready
@@ -139,23 +203,6 @@ impl iced_wgpu::Primitive for VideoFrame {
             return;
         };
 
-        // encoder.copy_buffer_to_texture(
-        //     wgpu::TexelCopyBufferInfo {
-        //         buffer: &video.buffer,
-        //         layout: wgpu::TexelCopyBufferLayout {
-        //             offset: 0,
-        //             bytes_per_row: Some(4 * self.size.width),
-        //             rows_per_image: Some(self.size.height),
-        //         },
-        //     },
-        //     wgpu::TexelCopyTextureInfo {
-        //         texture: &video.texture,
-        //         mip_level: 0,
-        //         origin: wgpu::Origin3d::ZERO,
-        //         aspect: wgpu::TextureAspect::All,
-        //     },
-        //     self.size,
-        // );
         let mut render_pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor {
             label: Some("iced-video-render-pass"),
             color_attachments: &[Some(wgpu::RenderPassColorAttachment {
@@ -186,36 +233,251 @@ impl iced_wgpu::Primitive for VideoFrame {
     }
 }
 
+/// NV12 or P010 are only supported in DX12 and Vulkan backends.
+/// While we can use vulkan with moltenvk on macos, I'd much rather use metal directly
 #[derive(Debug)]
-pub struct VideoTextures {
+pub struct VideoTexture {
+    y: wgpu::Texture,
+    uv: wgpu::Texture,
+    size: wgpu::Extent3d,
+    pixel_format: gst::VideoFormat,
+}
+
+impl VideoTexture {
+    pub fn size(&self) -> wgpu::Extent3d {
+        self.size
+    }
+
+    pub fn new(
+        label: &str,
+        size: wgpu::Extent3d,
+        device: &wgpu::Device,
+        format: wgpu::TextureFormat,
+        video_format: VideoFormat,
+    ) -> Self {
+        let y_texture = device.create_texture(&wgpu::TextureDescriptor {
+            label: Some(&format!("{}-y", label)),
+            size: wgpu::Extent3d {
+                width: size.width,
+                height: size.height,
+                depth_or_array_layers: 1,
+            },
+            mip_level_count: 1,
+            sample_count: 1,
+            dimension: wgpu::TextureDimension::D2,
+            format: wgpu::TextureFormat::R8Unorm,
+            usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
+            view_formats: &[],
+        });
+        let uv_texture = device.create_texture(&wgpu::TextureDescriptor {
+            label: Some(&format!("{}-uv", label)),
+            size: wgpu::Extent3d {
+                width: size.width / 2,
+                height: size.height / 2,
+                depth_or_array_layers: 1,
+            },
+            mip_level_count: 1,
+            sample_count: 1,
+            dimension: wgpu::TextureDimension::D2,
+            format: wgpu::TextureFormat::Rg8Unorm,
+            usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
+            view_formats: &[],
+        });
+        VideoTexture {
+            y: y_texture,
+            uv: uv_texture,
+            size,
+            pixel_format: VideoFormat::Unknown,
+        }
+    }
+
+    pub fn format(&self) -> wgpu::TextureFormat {
+        match self {
+            VideoTexture::NV12(_) => wgpu::TextureFormat::NV12,
+            VideoTexture::P010(_) => wgpu::TextureFormat::P010,
+            VideoTexture::Composite { y, uv } => {
+                todo!()
+                // if y.format().is_wide() {
+                //     wgpu::TextureFormat::P010
+                // } else {
+                //     wgpu::TextureFormat::NV12
+                // }
+            }
+        }
+    }
+
+    pub fn y_texture(&self) -> wgpu::TextureView {
+        match self {
+            VideoTexture::NV12(nv12) => nv12.create_view(&wgpu::TextureViewDescriptor {
+                label: Some("iced-video-texture-view-y-nv12"),
+                format: Some(wgpu::TextureFormat::R8Unorm),
+                ..Default::default()
+            }),
+            VideoTexture::P010(p010) => p010.create_view(&wgpu::TextureViewDescriptor {
+                label: Some("iced-video-texture-view-y-p010"),
+                format: Some(wgpu::TextureFormat::R16Unorm),
+                ..Default::default()
+            }),
+            VideoTexture::Composite { y, .. } => {
+                y.create_view(&wgpu::TextureViewDescriptor::default())
+            }
+        }
+    }
+
+    pub fn uv_texture(&self) -> wgpu::TextureView {
+        match self {
+            VideoTexture::NV12(nv12) => nv12.create_view(&wgpu::TextureViewDescriptor {
+                label: Some("iced-video-texture-view-uv-nv12"),
+                format: Some(wgpu::TextureFormat::Rg8Unorm),
+                ..Default::default()
+            }),
+            VideoTexture::P010(p010) => p010.create_view(&wgpu::TextureViewDescriptor {
+                label: Some("iced-video-texture-view-uv-p010"),
+                format: Some(wgpu::TextureFormat::Rg16Unorm),
+                ..Default::default()
+            }),
+            VideoTexture::Composite { uv, .. } => {
+                uv.create_view(&wgpu::TextureViewDescriptor::default())
+            }
+        }
+    }
+
+    pub fn resize(&self, name: &str, new_size: wgpu::Extent3d, device: &wgpu::Device) -> Self {
+        VideoTexture::new(name, new_size, device, self.format())
+    }
+
+    /// This assumes that the data is laid out correctly for the texture format.
+    pub fn write_texture(&self, data: &[u8], queue: &wgpu::Queue) {
+        match self {
+            VideoTexture::NV12(nv12) => {
+                queue.write_texture(
+                    wgpu::TexelCopyTextureInfo {
+                        texture: nv12,
+                        mip_level: 0,
+                        origin: wgpu::Origin3d::ZERO,
+                        aspect: wgpu::TextureAspect::All,
+                    },
+                    data,
+                    wgpu::TexelCopyBufferLayout {
+                        offset: 0,
+                        bytes_per_row: Some(nv12.size().width * 3),
+                        rows_per_image: Some(nv12.size().height),
+                    },
+                    nv12.size(),
+                );
+            }
+            VideoTexture::P010(p010) => {
+                dbg!(&p010.size());
+                dbg!(data.len());
+                queue.write_texture(
+                    wgpu::TexelCopyTextureInfo {
+                        texture: p010,
+                        mip_level: 0,
+                        origin: wgpu::Origin3d::ZERO,
+                        aspect: wgpu::TextureAspect::All,
+                    },
+                    data,
+                    wgpu::TexelCopyBufferLayout {
+                        offset: 0,
+                        bytes_per_row: Some(p010.size().width * 3),
+                        rows_per_image: Some(p010.size().height),
+                    },
+                    p010.size(),
+                );
+            }
+            VideoTexture::Composite { y, uv } => {
+                let y_size = wgpu::Extent3d {
+                    width: y.size().width,
+                    height: y.size().height,
+                    depth_or_array_layers: 1,
+                };
+                let uv_size = wgpu::Extent3d {
+                    width: uv.size().width,
+                    height: uv.size().height,
+                    depth_or_array_layers: 1,
+                };
+                let y_data_size = (y_size.width * y_size.height) as usize;
+                let uv_data_size = (uv_size.width * uv_size.height * 2) as usize; // UV is interleaved
+
+                queue.write_texture(
+                    wgpu::TexelCopyTextureInfo {
+                        texture: y,
+                        mip_level: 0,
+                        origin: wgpu::Origin3d::ZERO,
+                        aspect: wgpu::TextureAspect::All,
+                    },
+                    &data[0..y_data_size],
+                    wgpu::TexelCopyBufferLayout {
+                        offset: 0,
+                        bytes_per_row: Some(y_size.width),
+                        rows_per_image: Some(y_size.height),
+                    },
+                    y_size,
+                );
+
+                queue.write_texture(
+                    wgpu::TexelCopyTextureInfo {
+                        texture: uv,
+                        mip_level: 0,
+                        origin: wgpu::Origin3d::ZERO,
+                        aspect: wgpu::TextureAspect::All,
+                    },
+                    &data[y_data_size..(y_data_size + uv_data_size)],
+                    wgpu::TexelCopyBufferLayout {
+                        offset: 0,
+                        bytes_per_row: Some(uv_size.width * 2),
+                        rows_per_image: Some(uv_size.height),
+                    },
+                    uv_size,
+                );
+            }
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct VideoFrameData {
     id: id::Id,
-    texture: wgpu::Texture,
-    buffer: wgpu::Buffer,
+    texture: VideoTexture,
     bind_group: wgpu::BindGroup,
+    conversion_matrix: wgpu::Buffer,
     ready: Arc<AtomicBool>,
 }
 
+impl VideoFrameData {
+    pub fn is_hdr(&self) -> bool {
+        self.texture.format().is_wide()
+    }
+    pub fn is_nv12(&self) -> bool {
+        matches!(self.texture.format(), wgpu::TextureFormat::NV12)
+    }
+    pub fn is_p010(&self) -> bool {
+        matches!(self.texture.format(), wgpu::TextureFormat::P010)
+    }
+}
+
 #[derive(Debug)]
 pub struct VideoPipeline {
     pipeline: wgpu::RenderPipeline,
     bind_group_layout: wgpu::BindGroupLayout,
     sampler: wgpu::Sampler,
-    videos: BTreeMap<id::Id, VideoTextures>,
     format: wgpu::TextureFormat,
+    videos: BTreeMap<id::Id, VideoFrameData>,
 }
 
-pub trait HdrTextureFormatExt {
-    fn is_hdr(&self) -> bool;
+pub trait WideTextureFormatExt {
+    fn is_wide(&self) -> bool;
 }
 
-impl HdrTextureFormatExt for wgpu::TextureFormat {
-    fn is_hdr(&self) -> bool {
+impl WideTextureFormatExt for wgpu::TextureFormat {
+    fn is_wide(&self) -> bool {
         matches!(
             self,
             wgpu::TextureFormat::Rgba16Float
                 | wgpu::TextureFormat::Rgba32Float
                 | wgpu::TextureFormat::Rgb10a2Unorm
                 | wgpu::TextureFormat::Rgb10a2Uint
+                | wgpu::TextureFormat::P010
         )
     }
 }
@@ -225,15 +487,14 @@ impl Pipeline for VideoPipeline {
     where
         Self: Sized,
     {
-        if format.is_hdr() {
+        if format.is_wide() {
             tracing::info!("HDR texture format detected: {:?}", format);
         }
-        let shader_passthrough =
-            device.create_shader_module(wgpu::include_wgsl!("shaders/passthrough.wgsl"));
 
         let bind_group_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
             label: Some("iced-video-texture-bind-group-layout"),
             entries: &[
+                // y
                 wgpu::BindGroupLayoutEntry {
                     binding: 0,
                     visibility: wgpu::ShaderStages::FRAGMENT,
@@ -244,15 +505,40 @@ impl Pipeline for VideoPipeline {
                     },
                     count: None,
                 },
+                // uv
                 wgpu::BindGroupLayoutEntry {
                     binding: 1,
                     visibility: wgpu::ShaderStages::FRAGMENT,
+                    ty: wgpu::BindingType::Texture {
+                        multisampled: false,
+                        view_dimension: wgpu::TextureViewDimension::D2,
+                        sample_type: wgpu::TextureSampleType::Float { filterable: true },
+                    },
+                    count: None,
+                },
+                // sampler
+                wgpu::BindGroupLayoutEntry {
+                    binding: 2,
+                    visibility: wgpu::ShaderStages::FRAGMENT,
                     ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering),
                     count: None,
                 },
+                // conversion matrix
+                wgpu::BindGroupLayoutEntry {
+                    binding: 3,
+                    visibility: wgpu::ShaderStages::FRAGMENT,
+                    ty: wgpu::BindingType::Buffer {
+                        ty: wgpu::BufferBindingType::Uniform,
+                        has_dynamic_offset: false,
+                        min_binding_size: None,
+                    },
+                    count: None,
+                },
             ],
         });
 
+        let shader_passthrough =
+            device.create_shader_module(wgpu::include_wgsl!("shaders/passthrough.wgsl"));
         let render_pipeline_layout =
             device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
                 label: Some("iced-video-render-pipeline-layout"),
@@ -273,7 +559,7 @@ impl Pipeline for VideoPipeline {
                 entry_point: Some("fs_main"),
                 targets: &[Some(wgpu::ColorTargetState {
                     format,
-                    blend: Some(wgpu::BlendState::ALPHA_BLENDING),
+                    blend: Some(wgpu::BlendState::REPLACE),
                     write_mask: wgpu::ColorWrites::ALL,
                 })],
                 compilation_options: wgpu::PipelineCompilationOptions::default(),

crates/iced-video/src/shaders/passthrough.wgsl

@@ -1,31 +1,52 @@
-// Vertex shader
+// struct VertexOutput {
+//     @builtin(position) clip_position: vec4f,
+//     @location(0) coords: vec2f,
+// }
+
+// struct VertexInput {
+//     // @location(0) position: vec3<f32>,
+//     // @location(1) tex_coords: vec2<f32>,
+// }
 
 struct VertexOutput {
     @builtin(position) clip_position: vec4<f32>,
     @location(0) tex_coords: vec2<f32>,
-};
+}
 
 @vertex
 fn vs_main(
-    @builtin(vertex_index) in_vertex_index: u32,
-) -> VertexOutput {
+    // model: VertexInput,
+) -> VertexOutput { 
     var out: VertexOutput;
-    let uv = vec2<f32>(f32((in_vertex_index << 1u) & 2u), f32(in_vertex_index & 2u));
-    out.clip_position = vec4<f32>(uv * 2.0 - 1.0, 0.0, 1.0);
-    out.clip_position.y = -out.clip_position.y;
-    out.tex_coords = uv;
+    out.tex_coords = vec2<f32>(0.0, 0.0);
+    out.clip_position = vec4<f32>(0,0,0, 1.0);
     return out;
 }
 
-// Fragment shader
+ 
 
-@group(0) @binding(0)
-var t_diffuse: texture_2d<f32>;
-@group(0) @binding(1)
-var s_diffuse: sampler;
+
+
+
+// @vertex
+// fn vs_main(@location(0) input: vec2f) -> VertexOutput {
+//     var out: VertexOutput;
+//     out.clip_position = vec4f(input, 0.0, 1.0);
+//     out.coords = input * 0.5 + vec2f(0.5, 0.5);
+//     return out;
+// }
+
+@group(0) @binding(0) var y_texture: texture_2d<f32>;
+@group(0) @binding(1) var uv_texture: texture_2d<f32>;
+@group(0) @binding(2) var texture_sampler: sampler;
+@group(0) @binding(3) var<uniform> rgb_primaries: mat3x3<f32>;
 
 @fragment
-fn fs_main(in: VertexOutput) -> @location(0) vec4<f32> {
-    return textureSample(t_diffuse, s_diffuse, in.tex_coords);
+fn fs_main(input: VertexOutput) -> @location(0) vec4<f32> {
+    let y = textureSample(y_texture,  texture_sampler, input.tex_coords).r;
+    let uv = textureSample(uv_texture, texture_sampler, input.tex_coords).rg;
+    let yuv = vec3f(y, uv);
+    let rgb = rgb_primaries * yuv;
+    return vec4f(rgb, 1.0);
 }