feat(iced-video): implement planar YUV texture support with HDR conversion matrices and update dependencies

Cargo.lock

@@ -3437,7 +3437,7 @@ dependencies = [
 [[package]]
 name = "iced"
 version = "0.14.0"
-source = "git+https://github.com/uttarayan21/iced?branch=0.14#5846d52983d7e2eecc478130ba6373f0c1f82c94"
+source = "git+https://github.com/uttarayan21/iced?branch=0.14#6fbe1ec83722c67cf7f43291254b357ffc5b6fb6"
 dependencies = [
  "iced_core",
  "iced_debug 0.14.0 (git+https://github.com/uttarayan21/iced?branch=0.14)",
@@ -3455,6 +3455,7 @@ dependencies = [
 name = "iced-video"
 version = "0.1.0"
 dependencies = [
+ "bytemuck",
  "error-stack",
  "futures-lite 2.6.1",
  "gst",
@@ -3466,12 +3467,13 @@ dependencies = [
  "thiserror 2.0.17",
  "tracing",
  "tracing-subscriber",
+ "wgpu",
 ]
 
 [[package]]
 name = "iced_beacon"
 version = "0.14.0"
-source = "git+https://github.com/uttarayan21/iced?branch=0.14#5846d52983d7e2eecc478130ba6373f0c1f82c94"
+source = "git+https://github.com/uttarayan21/iced?branch=0.14#6fbe1ec83722c67cf7f43291254b357ffc5b6fb6"
 dependencies = [
  "bincode 1.3.3",
  "futures",
@@ -3486,7 +3488,7 @@ dependencies = [
 [[package]]
 name = "iced_core"
 version = "0.14.0"
-source = "git+https://github.com/uttarayan21/iced?branch=0.14#5846d52983d7e2eecc478130ba6373f0c1f82c94"
+source = "git+https://github.com/uttarayan21/iced?branch=0.14#6fbe1ec83722c67cf7f43291254b357ffc5b6fb6"
 dependencies = [
  "bitflags 2.10.0",
  "bytes",
@@ -3515,7 +3517,7 @@ dependencies = [
 [[package]]
 name = "iced_debug"
 version = "0.14.0"
-source = "git+https://github.com/uttarayan21/iced?branch=0.14#5846d52983d7e2eecc478130ba6373f0c1f82c94"
+source = "git+https://github.com/uttarayan21/iced?branch=0.14#6fbe1ec83722c67cf7f43291254b357ffc5b6fb6"
 dependencies = [
  "cargo-hot-protocol",
  "iced_beacon",
@@ -3527,7 +3529,7 @@ dependencies = [
 [[package]]
 name = "iced_devtools"
 version = "0.14.0"
-source = "git+https://github.com/uttarayan21/iced?branch=0.14#5846d52983d7e2eecc478130ba6373f0c1f82c94"
+source = "git+https://github.com/uttarayan21/iced?branch=0.14#6fbe1ec83722c67cf7f43291254b357ffc5b6fb6"
 dependencies = [
  "iced_debug 0.14.0 (git+https://github.com/uttarayan21/iced?branch=0.14)",
  "iced_program 0.14.0 (git+https://github.com/uttarayan21/iced?branch=0.14)",
@@ -3538,7 +3540,7 @@ dependencies = [
 [[package]]
 name = "iced_futures"
 version = "0.14.0"
-source = "git+https://github.com/uttarayan21/iced?branch=0.14#5846d52983d7e2eecc478130ba6373f0c1f82c94"
+source = "git+https://github.com/uttarayan21/iced?branch=0.14#6fbe1ec83722c67cf7f43291254b357ffc5b6fb6"
 dependencies = [
  "futures",
  "iced_core",
@@ -3571,7 +3573,7 @@ dependencies = [
 [[package]]
 name = "iced_graphics"
 version = "0.14.0"
-source = "git+https://github.com/uttarayan21/iced?branch=0.14#5846d52983d7e2eecc478130ba6373f0c1f82c94"
+source = "git+https://github.com/uttarayan21/iced?branch=0.14#6fbe1ec83722c67cf7f43291254b357ffc5b6fb6"
 dependencies = [
  "bitflags 2.10.0",
  "bytemuck",
@@ -3602,7 +3604,7 @@ dependencies = [
 [[package]]
 name = "iced_program"
 version = "0.14.0"
-source = "git+https://github.com/uttarayan21/iced?branch=0.14#5846d52983d7e2eecc478130ba6373f0c1f82c94"
+source = "git+https://github.com/uttarayan21/iced?branch=0.14#6fbe1ec83722c67cf7f43291254b357ffc5b6fb6"
 dependencies = [
  "iced_graphics 0.14.0 (git+https://github.com/uttarayan21/iced?branch=0.14)",
  "iced_runtime 0.14.0 (git+https://github.com/uttarayan21/iced?branch=0.14)",
@@ -3611,7 +3613,7 @@ dependencies = [
 [[package]]
 name = "iced_renderer"
 version = "0.14.0"
-source = "git+https://github.com/uttarayan21/iced?branch=0.14#5846d52983d7e2eecc478130ba6373f0c1f82c94"
+source = "git+https://github.com/uttarayan21/iced?branch=0.14#6fbe1ec83722c67cf7f43291254b357ffc5b6fb6"
 dependencies = [
  "iced_graphics 0.14.0 (git+https://github.com/uttarayan21/iced?branch=0.14)",
  "iced_tiny_skia",
@@ -3636,7 +3638,7 @@ dependencies = [
 [[package]]
 name = "iced_runtime"
 version = "0.14.0"
-source = "git+https://github.com/uttarayan21/iced?branch=0.14#5846d52983d7e2eecc478130ba6373f0c1f82c94"
+source = "git+https://github.com/uttarayan21/iced?branch=0.14#6fbe1ec83722c67cf7f43291254b357ffc5b6fb6"
 dependencies = [
  "bytes",
  "iced_core",
@@ -3649,7 +3651,7 @@ dependencies = [
 [[package]]
 name = "iced_tiny_skia"
 version = "0.14.0"
-source = "git+https://github.com/uttarayan21/iced?branch=0.14#5846d52983d7e2eecc478130ba6373f0c1f82c94"
+source = "git+https://github.com/uttarayan21/iced?branch=0.14#6fbe1ec83722c67cf7f43291254b357ffc5b6fb6"
 dependencies = [
  "bytemuck",
  "cosmic-text 0.15.0",
@@ -3665,7 +3667,7 @@ dependencies = [
 [[package]]
 name = "iced_wgpu"
 version = "0.14.0"
-source = "git+https://github.com/uttarayan21/iced?branch=0.14#5846d52983d7e2eecc478130ba6373f0c1f82c94"
+source = "git+https://github.com/uttarayan21/iced?branch=0.14#6fbe1ec83722c67cf7f43291254b357ffc5b6fb6"
 dependencies = [
  "bitflags 2.10.0",
  "bytemuck",
@@ -3685,7 +3687,7 @@ dependencies = [
 [[package]]
 name = "iced_widget"
 version = "0.14.2"
-source = "git+https://github.com/uttarayan21/iced?branch=0.14#5846d52983d7e2eecc478130ba6373f0c1f82c94"
+source = "git+https://github.com/uttarayan21/iced?branch=0.14#6fbe1ec83722c67cf7f43291254b357ffc5b6fb6"
 dependencies = [
  "iced_renderer",
  "log",
@@ -3716,7 +3718,7 @@ dependencies = [
 [[package]]
 name = "iced_winit"
 version = "0.14.0"
-source = "git+https://github.com/uttarayan21/iced?branch=0.14#5846d52983d7e2eecc478130ba6373f0c1f82c94"
+source = "git+https://github.com/uttarayan21/iced?branch=0.14#6fbe1ec83722c67cf7f43291254b357ffc5b6fb6"
 dependencies = [
  "iced_debug 0.14.0 (git+https://github.com/uttarayan21/iced?branch=0.14)",
  "iced_program 0.14.0 (git+https://github.com/uttarayan21/iced?branch=0.14)",
@@ -4043,6 +4045,7 @@ name = "jello"
 version = "0.1.0"
 dependencies = [
  "api",
+ "bytemuck",
  "clap",
  "clap-verbosity-flag",
  "clap_complete",
@@ -8781,7 +8784,7 @@ version = "0.1.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
 dependencies = [
- "windows-sys 0.61.2",
+ "windows-sys 0.48.0",
 ]
 
 [[package]]

Cargo.toml

@@ -32,6 +32,7 @@ license = "MIT"
 
 [dependencies]
 api = { version = "0.1.0", path = "api" }
+bytemuck = { version = "1.24.0", features = ["derive"] }
 clap = { version = "4.5", features = ["derive"] }
 clap-verbosity-flag = { version = "3.0.4", features = ["tracing"] }
 clap_complete = "4.5"

README.md

@@ -63,3 +63,47 @@ In the shader the components get uniformly normalized from [0..=1023] integer to
 Videos however are generally not stored in this format or any rgb format in general because it is not as efficient for (lossy) compression as YUV formats.
 
 Right now I don't want to deal with yuv formats so I'll use gstreamer caps to convert the video into `Rgba10a2` format
+
+
+## Pixel formats and Planes
+Dated: Sun Jan  4 09:09:16 AM IST 2026
+| value | count | quantile | percentage | frequency |
+| --- | --- | --- | --- | --- |
+| yuv420p | 1815 | 0.5067001675041876 | 50.67% | ************************************************** |
+| yuv420p10le | 1572 | 0.4388609715242881 | 43.89% | ******************************************* |
+| yuvj420p | 171 | 0.04773869346733668 | 4.77% | **** |
+| rgba | 14 | 0.003908431044109436 | 0.39% |  |
+| yuvj444p | 10 | 0.0027917364600781687 | 0.28% |  |
+
+For all of my media collection these are the pixel formats for all the videos
+
+### RGBA
+Pretty self evident   
+8 channels for each of R, G, B and A  
+Hopefully shouldn't be too hard to make a function or possibly a lut that takes data from rgba and maps it to Rgb10a2Unorm  
+
+```mermaid
+packet
++8: "R"
++8: "G"
++8: "B"
++8: "A"
+```
+
+
+### YUV
+[All YUV formats](https://learn.microsoft.com/en-us/windows/win32/medfound/recommended-8-bit-yuv-formats-for-video-rendering#surface-definitions)  
+[10 and 16 bit yuv formats](https://learn.microsoft.com/en-us/windows/win32/medfound/10-bit-and-16-bit-yuv-video-formats)
+
+Y -> Luminance
+U,V -> Chrominance
+
+p -> Planar
+sp -> semi planar
+
+j -> full range 
+
+planar formats have each of the channels in a contiguous array one after another
+in semi-planar formats the y channel is seperate and uv channels are interleaved
+
+

crates/iced-video/Cargo.toml

@@ -4,6 +4,7 @@ version = "0.1.0"
 edition = "2024"
 
 [dependencies]
+bytemuck = "1.24.0"
 error-stack = "0.6.0"
 futures-lite = "2.6.1"
 gst.workspace = true
@@ -13,6 +14,7 @@ iced_renderer = { version = "0.14.0", features = ["iced_wgpu"] }
 iced_wgpu = { version = "0.14.0" }
 thiserror = "2.0.17"
 tracing = "0.1.43"
+wgpu = { version = "27.0.1", features = ["vulkan"] }
 
 [dev-dependencies]
 iced.workspace = true

crates/iced-video/src/primitive.rs

@@ -1,9 +1,65 @@
 use crate::id;
+use gst::videoconvertscale::VideoFormat;
 use iced_wgpu::primitive::Pipeline;
 use iced_wgpu::wgpu;
 use std::collections::BTreeMap;
 use std::sync::{Arc, Mutex, atomic::AtomicBool};
 
+#[derive(Clone, Copy, Debug, bytemuck::Zeroable, bytemuck::Pod)]
+#[repr(transparent)]
+pub struct ConversionMatrix {
+    matrix: [[f32; 4]; 4],
+}
+
+// impl ConversionMatrix {
+//     pub fn desc() -> wgpu::VertexBufferLayout<'static> {
+//         wgpu::VertexBufferLayout {
+//             array_stride: core::mem::size_of::<ConversionMatrix>() as wgpu::BufferAddress,
+//             step_mode: wgpu::VertexStepMode::Vertex,
+//             attributes: &[
+//                 wgpu::VertexAttribute {
+//                     offset: 0,
+//                     shader_location: 0,
+//                     format: wgpu::VertexFormat::Float32x4,
+//                 },
+//                 wgpu::VertexAttribute {
+//                     offset: 16,
+//                     shader_location: 1,
+//                     format: wgpu::VertexFormat::Float32x4,
+//                 },
+//                 wgpu::VertexAttribute {
+//                     offset: 32,
+//                     shader_location: 2,
+//                     format: wgpu::VertexFormat::Float32x4,
+//                 },
+//                 wgpu::VertexAttribute {
+//                     offset: 48,
+//                     shader_location: 3,
+//                     format: wgpu::VertexFormat::Float32x4,
+//                 },
+//             ],
+//         }
+//     }
+// }
+
+pub const BT2020_TO_RGB: ConversionMatrix = ConversionMatrix {
+    matrix: [
+        [1.1684, 0.0000, 1.6836, -0.9122],
+        [1.1684, -0.1873, -0.6520, 0.3015],
+        [1.1684, 2.1482, 0.0000, -1.1322],
+        [0.0, 0.0, 0.0, 1.0],
+    ],
+};
+
+pub const BT709_TO_RGB: ConversionMatrix = ConversionMatrix {
+    matrix: [
+        [1.1644, 0.0000, 1.7927, -0.9729],
+        [1.1644, -0.2132, -0.5329, 0.3015],
+        [1.1644, 2.1124, 0.0000, -1.1334],
+        [0.0, 0.0, 0.0, 1.0],
+    ],
+};
+
 #[derive(Debug)]
 pub struct VideoFrame {
     pub id: id::Id,
@@ -24,73 +80,78 @@ impl iced_wgpu::Primitive for VideoFrame {
         viewport: &iced_wgpu::graphics::Viewport,
     ) {
         let video = pipeline.videos.entry(self.id.clone()).or_insert_with(|| {
-            let texture = device.create_texture(&wgpu::TextureDescriptor {
+            let texture =
-                label: Some("iced-video-texture"),
+                VideoTexture::new("iced-video-texture", self.size, device, pipeline.format);
-                size: self.size,
+            let conversion_matrix = if texture.format().is_wide() {
-                mip_level_count: 1,
+                BT2020_TO_RGB
-                sample_count: 1,
+            } else {
-                dimension: wgpu::TextureDimension::D2,
+                BT709_TO_RGB
-                format: pipeline.format,
+            };
-                usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
-                view_formats: &[],
-            });
             let buffer = device.create_buffer(&wgpu::BufferDescriptor {
-                label: Some("iced-video-buffer"),
+                label: Some("iced-video-conversion-matrix-buffer"),
-                size: (self.size.width * self.size.height * 4) as u64,
+                usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST,
-                usage: wgpu::BufferUsages::COPY_SRC | wgpu::BufferUsages::COPY_DST,
+                size: core::mem::size_of::<ConversionMatrix>() as wgpu::BufferAddress,
                 mapped_at_creation: false,
             });
+
             let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor {
                 label: Some("iced-video-texture-bind-group"),
                 layout: &pipeline.bind_group_layout,
                 entries: &[
                     wgpu::BindGroupEntry {
                         binding: 0,
-                        resource: wgpu::BindingResource::TextureView(
+                        resource: wgpu::BindingResource::TextureView(&texture.y_texture()),
-                            &texture.create_view(&wgpu::TextureViewDescriptor::default()),
-                        ),
                     },
                     wgpu::BindGroupEntry {
                         binding: 1,
+                        resource: wgpu::BindingResource::TextureView(&texture.uv_texture()),
+                    },
+                    wgpu::BindGroupEntry {
+                        binding: 2,
                         resource: wgpu::BindingResource::Sampler(&pipeline.sampler),
                     },
+                    wgpu::BindGroupEntry {
+                        binding: 3,
+                        resource: wgpu::BindingResource::Buffer(buffer.as_entire_buffer_binding()),
+                    },
                 ],
             });
-            VideoTextures {
+
+            VideoFrameData {
                 id: self.id.clone(),
                 texture,
-                buffer,
+                conversion_matrix: buffer,
                 bind_group,
                 ready: Arc::clone(&self.ready),
             }
         });
-        // dbg!(&self.size, video.texture.size());
         if self.size != video.texture.size() {
-            // Resize the texture if the size has changed.
+            let new_texture = video
-            let new_texture = device.create_texture(&wgpu::TextureDescriptor {
+                .texture
-                label: Some("iced-video-texture-resized"),
+                .resize("iced-video-texture-resized", self.size, device);
-                size: self.size,
+
-                mip_level_count: 1,
-                sample_count: 1,
-                dimension: wgpu::TextureDimension::D2,
-                format: pipeline.format,
-                usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
-                view_formats: &[],
-            });
             let new_bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor {
-                label: Some("iced-video-texture-bind-group-resized"),
+                label: Some("iced-video-texture-bind-group"),
                 layout: &pipeline.bind_group_layout,
                 entries: &[
                     wgpu::BindGroupEntry {
                         binding: 0,
-                        resource: wgpu::BindingResource::TextureView(
+                        resource: wgpu::BindingResource::TextureView(&new_texture.y_texture()),
-                            &new_texture.create_view(&wgpu::TextureViewDescriptor::default()),
-                        ),
                     },
                     wgpu::BindGroupEntry {
                         binding: 1,
+                        resource: wgpu::BindingResource::TextureView(&new_texture.uv_texture()),
+                    },
+                    wgpu::BindGroupEntry {
+                        binding: 2,
                         resource: wgpu::BindingResource::Sampler(&pipeline.sampler),
                     },
+                    wgpu::BindGroupEntry {
+                        binding: 3,
+                        resource: wgpu::BindingResource::Buffer(
+                            video.conversion_matrix.as_entire_buffer_binding(),
+                        ),
+                    },
                 ],
             });
             video.texture = new_texture;
@@ -106,21 +167,24 @@ impl iced_wgpu::Primitive for VideoFrame {
                 .map_readable()
                 .expect("BUG: Failed to map gst::Buffer readable");
             // queue.write_buffer(&video.buffer, 0, &data);
-            queue.write_texture(
+
-                wgpu::TexelCopyTextureInfo {
+            video.texture.write_texture(&data, queue);
-                    texture: &video.texture,
+            // queue.write_texture(
-                    mip_level: 0,
+            //     wgpu::TexelCopyTextureInfo {
-                    origin: wgpu::Origin3d::ZERO,
+            //         texture: &video.texture,
-                    aspect: wgpu::TextureAspect::All,
+            //         mip_level: 0,
-                },
+            //         origin: wgpu::Origin3d::ZERO,
-                &data,
+            //         aspect: wgpu::TextureAspect::All,
-                wgpu::TexelCopyBufferLayout {
+            //     },
-                    offset: 0,
+            //     &data,
-                    bytes_per_row: Some(4 * self.size.width),
+            //     wgpu::TexelCopyBufferLayout {
-                    rows_per_image: Some(self.size.height),
+            //         offset: 0,
-                },
+            //         bytes_per_row: Some(4 * self.size.width),
-                self.size,
+            //         rows_per_image: Some(self.size.height),
-            );
+            //     },
+            //     self.size,
+            // );
+
             drop(data);
             video
                 .ready
@@ -139,23 +203,6 @@ impl iced_wgpu::Primitive for VideoFrame {
             return;
         };
 
-        // encoder.copy_buffer_to_texture(
-        //     wgpu::TexelCopyBufferInfo {
-        //         buffer: &video.buffer,
-        //         layout: wgpu::TexelCopyBufferLayout {
-        //             offset: 0,
-        //             bytes_per_row: Some(4 * self.size.width),
-        //             rows_per_image: Some(self.size.height),
-        //         },
-        //     },
-        //     wgpu::TexelCopyTextureInfo {
-        //         texture: &video.texture,
-        //         mip_level: 0,
-        //         origin: wgpu::Origin3d::ZERO,
-        //         aspect: wgpu::TextureAspect::All,
-        //     },
-        //     self.size,
-        // );
         let mut render_pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor {
             label: Some("iced-video-render-pass"),
             color_attachments: &[Some(wgpu::RenderPassColorAttachment {
@@ -186,36 +233,251 @@ impl iced_wgpu::Primitive for VideoFrame {
     }
 }
 
+/// NV12 or P010 are only supported in DX12 and Vulkan backends.
+/// While we can use vulkan with moltenvk on macos, I'd much rather use metal directly
 #[derive(Debug)]
-pub struct VideoTextures {
+pub struct VideoTexture {
+    y: wgpu::Texture,
+    uv: wgpu::Texture,
+    size: wgpu::Extent3d,
+    pixel_format: gst::VideoFormat,
+}
+
+impl VideoTexture {
+    pub fn size(&self) -> wgpu::Extent3d {
+        self.size
+    }
+
+    pub fn new(
+        label: &str,
+        size: wgpu::Extent3d,
+        device: &wgpu::Device,
+        format: wgpu::TextureFormat,
+        video_format: VideoFormat,
+    ) -> Self {
+        let y_texture = device.create_texture(&wgpu::TextureDescriptor {
+            label: Some(&format!("{}-y", label)),
+            size: wgpu::Extent3d {
+                width: size.width,
+                height: size.height,
+                depth_or_array_layers: 1,
+            },
+            mip_level_count: 1,
+            sample_count: 1,
+            dimension: wgpu::TextureDimension::D2,
+            format: wgpu::TextureFormat::R8Unorm,
+            usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
+            view_formats: &[],
+        });
+        let uv_texture = device.create_texture(&wgpu::TextureDescriptor {
+            label: Some(&format!("{}-uv", label)),
+            size: wgpu::Extent3d {
+                width: size.width / 2,
+                height: size.height / 2,
+                depth_or_array_layers: 1,
+            },
+            mip_level_count: 1,
+            sample_count: 1,
+            dimension: wgpu::TextureDimension::D2,
+            format: wgpu::TextureFormat::Rg8Unorm,
+            usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
+            view_formats: &[],
+        });
+        VideoTexture {
+            y: y_texture,
+            uv: uv_texture,
+            size,
+            pixel_format: VideoFormat::Unknown,
+        }
+    }
+
+    pub fn format(&self) -> wgpu::TextureFormat {
+        match self {
+            VideoTexture::NV12(_) => wgpu::TextureFormat::NV12,
+            VideoTexture::P010(_) => wgpu::TextureFormat::P010,
+            VideoTexture::Composite { y, uv } => {
+                todo!()
+                // if y.format().is_wide() {
+                //     wgpu::TextureFormat::P010
+                // } else {
+                //     wgpu::TextureFormat::NV12
+                // }
+            }
+        }
+    }
+
+    pub fn y_texture(&self) -> wgpu::TextureView {
+        match self {
+            VideoTexture::NV12(nv12) => nv12.create_view(&wgpu::TextureViewDescriptor {
+                label: Some("iced-video-texture-view-y-nv12"),
+                format: Some(wgpu::TextureFormat::R8Unorm),
+                ..Default::default()
+            }),
+            VideoTexture::P010(p010) => p010.create_view(&wgpu::TextureViewDescriptor {
+                label: Some("iced-video-texture-view-y-p010"),
+                format: Some(wgpu::TextureFormat::R16Unorm),
+                ..Default::default()
+            }),
+            VideoTexture::Composite { y, .. } => {
+                y.create_view(&wgpu::TextureViewDescriptor::default())
+            }
+        }
+    }
+
+    pub fn uv_texture(&self) -> wgpu::TextureView {
+        match self {
+            VideoTexture::NV12(nv12) => nv12.create_view(&wgpu::TextureViewDescriptor {
+                label: Some("iced-video-texture-view-uv-nv12"),
+                format: Some(wgpu::TextureFormat::Rg8Unorm),
+                ..Default::default()
+            }),
+            VideoTexture::P010(p010) => p010.create_view(&wgpu::TextureViewDescriptor {
+                label: Some("iced-video-texture-view-uv-p010"),
+                format: Some(wgpu::TextureFormat::Rg16Unorm),
+                ..Default::default()
+            }),
+            VideoTexture::Composite { uv, .. } => {
+                uv.create_view(&wgpu::TextureViewDescriptor::default())
+            }
+        }
+    }
+
+    pub fn resize(&self, name: &str, new_size: wgpu::Extent3d, device: &wgpu::Device) -> Self {
+        VideoTexture::new(name, new_size, device, self.format())
+    }
+
+    /// This assumes that the data is laid out correctly for the texture format.
+    pub fn write_texture(&self, data: &[u8], queue: &wgpu::Queue) {
+        match self {
+            VideoTexture::NV12(nv12) => {
+                queue.write_texture(
+                    wgpu::TexelCopyTextureInfo {
+                        texture: nv12,
+                        mip_level: 0,
+                        origin: wgpu::Origin3d::ZERO,
+                        aspect: wgpu::TextureAspect::All,
+                    },
+                    data,
+                    wgpu::TexelCopyBufferLayout {
+                        offset: 0,
+                        bytes_per_row: Some(nv12.size().width * 3),
+                        rows_per_image: Some(nv12.size().height),
+                    },
+                    nv12.size(),
+                );
+            }
+            VideoTexture::P010(p010) => {
+                dbg!(&p010.size());
+                dbg!(data.len());
+                queue.write_texture(
+                    wgpu::TexelCopyTextureInfo {
+                        texture: p010,
+                        mip_level: 0,
+                        origin: wgpu::Origin3d::ZERO,
+                        aspect: wgpu::TextureAspect::All,
+                    },
+                    data,
+                    wgpu::TexelCopyBufferLayout {
+                        offset: 0,
+                        bytes_per_row: Some(p010.size().width * 3),
+                        rows_per_image: Some(p010.size().height),
+                    },
+                    p010.size(),
+                );
+            }
+            VideoTexture::Composite { y, uv } => {
+                let y_size = wgpu::Extent3d {
+                    width: y.size().width,
+                    height: y.size().height,
+                    depth_or_array_layers: 1,
+                };
+                let uv_size = wgpu::Extent3d {
+                    width: uv.size().width,
+                    height: uv.size().height,
+                    depth_or_array_layers: 1,
+                };
+                let y_data_size = (y_size.width * y_size.height) as usize;
+                let uv_data_size = (uv_size.width * uv_size.height * 2) as usize; // UV is interleaved
+
+                queue.write_texture(
+                    wgpu::TexelCopyTextureInfo {
+                        texture: y,
+                        mip_level: 0,
+                        origin: wgpu::Origin3d::ZERO,
+                        aspect: wgpu::TextureAspect::All,
+                    },
+                    &data[0..y_data_size],
+                    wgpu::TexelCopyBufferLayout {
+                        offset: 0,
+                        bytes_per_row: Some(y_size.width),
+                        rows_per_image: Some(y_size.height),
+                    },
+                    y_size,
+                );
+
+                queue.write_texture(
+                    wgpu::TexelCopyTextureInfo {
+                        texture: uv,
+                        mip_level: 0,
+                        origin: wgpu::Origin3d::ZERO,
+                        aspect: wgpu::TextureAspect::All,
+                    },
+                    &data[y_data_size..(y_data_size + uv_data_size)],
+                    wgpu::TexelCopyBufferLayout {
+                        offset: 0,
+                        bytes_per_row: Some(uv_size.width * 2),
+                        rows_per_image: Some(uv_size.height),
+                    },
+                    uv_size,
+                );
+            }
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct VideoFrameData {
     id: id::Id,
-    texture: wgpu::Texture,
+    texture: VideoTexture,
-    buffer: wgpu::Buffer,
     bind_group: wgpu::BindGroup,
+    conversion_matrix: wgpu::Buffer,
     ready: Arc<AtomicBool>,
 }
 
+impl VideoFrameData {
+    pub fn is_hdr(&self) -> bool {
+        self.texture.format().is_wide()
+    }
+    pub fn is_nv12(&self) -> bool {
+        matches!(self.texture.format(), wgpu::TextureFormat::NV12)
+    }
+    pub fn is_p010(&self) -> bool {
+        matches!(self.texture.format(), wgpu::TextureFormat::P010)
+    }
+}
+
 #[derive(Debug)]
 pub struct VideoPipeline {
     pipeline: wgpu::RenderPipeline,
     bind_group_layout: wgpu::BindGroupLayout,
     sampler: wgpu::Sampler,
-    videos: BTreeMap<id::Id, VideoTextures>,
     format: wgpu::TextureFormat,
+    videos: BTreeMap<id::Id, VideoFrameData>,
 }
 
-pub trait HdrTextureFormatExt {
+pub trait WideTextureFormatExt {
-    fn is_hdr(&self) -> bool;
+    fn is_wide(&self) -> bool;
 }
 
-impl HdrTextureFormatExt for wgpu::TextureFormat {
+impl WideTextureFormatExt for wgpu::TextureFormat {
-    fn is_hdr(&self) -> bool {
+    fn is_wide(&self) -> bool {
         matches!(
             self,
             wgpu::TextureFormat::Rgba16Float
                 | wgpu::TextureFormat::Rgba32Float
                 | wgpu::TextureFormat::Rgb10a2Unorm
                 | wgpu::TextureFormat::Rgb10a2Uint
+                | wgpu::TextureFormat::P010
         )
     }
 }
@@ -225,15 +487,14 @@ impl Pipeline for VideoPipeline {
     where
         Self: Sized,
     {
-        if format.is_hdr() {
+        if format.is_wide() {
             tracing::info!("HDR texture format detected: {:?}", format);
         }
-        let shader_passthrough =
-            device.create_shader_module(wgpu::include_wgsl!("shaders/passthrough.wgsl"));
 
         let bind_group_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
             label: Some("iced-video-texture-bind-group-layout"),
             entries: &[
+                // y
                 wgpu::BindGroupLayoutEntry {
                     binding: 0,
                     visibility: wgpu::ShaderStages::FRAGMENT,
@@ -244,15 +505,40 @@ impl Pipeline for VideoPipeline {
                     },
                     count: None,
                 },
+                // uv
                 wgpu::BindGroupLayoutEntry {
                     binding: 1,
                     visibility: wgpu::ShaderStages::FRAGMENT,
+                    ty: wgpu::BindingType::Texture {
+                        multisampled: false,
+                        view_dimension: wgpu::TextureViewDimension::D2,
+                        sample_type: wgpu::TextureSampleType::Float { filterable: true },
+                    },
+                    count: None,
+                },
+                // sampler
+                wgpu::BindGroupLayoutEntry {
+                    binding: 2,
+                    visibility: wgpu::ShaderStages::FRAGMENT,
                     ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering),
                     count: None,
                 },
+                // conversion matrix
+                wgpu::BindGroupLayoutEntry {
+                    binding: 3,
+                    visibility: wgpu::ShaderStages::FRAGMENT,
+                    ty: wgpu::BindingType::Buffer {
+                        ty: wgpu::BufferBindingType::Uniform,
+                        has_dynamic_offset: false,
+                        min_binding_size: None,
+                    },
+                    count: None,
+                },
             ],
         });
 
+        let shader_passthrough =
+            device.create_shader_module(wgpu::include_wgsl!("shaders/passthrough.wgsl"));
         let render_pipeline_layout =
             device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
                 label: Some("iced-video-render-pipeline-layout"),
@@ -273,7 +559,7 @@ impl Pipeline for VideoPipeline {
                 entry_point: Some("fs_main"),
                 targets: &[Some(wgpu::ColorTargetState {
                     format,
-                    blend: Some(wgpu::BlendState::ALPHA_BLENDING),
+                    blend: Some(wgpu::BlendState::REPLACE),
                     write_mask: wgpu::ColorWrites::ALL,
                 })],
                 compilation_options: wgpu::PipelineCompilationOptions::default(),

crates/iced-video/src/shaders/passthrough.wgsl

@@ -1,31 +1,52 @@
-// Vertex shader
+// struct VertexOutput {
+//     @builtin(position) clip_position: vec4f,
+//     @location(0) coords: vec2f,
+// }
+
+// struct VertexInput {
+//     // @location(0) position: vec3<f32>,
+//     // @location(1) tex_coords: vec2<f32>,
+// }
 
 struct VertexOutput {
     @builtin(position) clip_position: vec4<f32>,
     @location(0) tex_coords: vec2<f32>,
-};
+}
 
 @vertex
 fn vs_main(
-    @builtin(vertex_index) in_vertex_index: u32,
+    // model: VertexInput,
 ) -> VertexOutput { 
     var out: VertexOutput;
-    let uv = vec2<f32>(f32((in_vertex_index << 1u) & 2u), f32(in_vertex_index & 2u));
+    out.tex_coords = vec2<f32>(0.0, 0.0);
-    out.clip_position = vec4<f32>(uv * 2.0 - 1.0, 0.0, 1.0);
+    out.clip_position = vec4<f32>(0,0,0, 1.0);
-    out.clip_position.y = -out.clip_position.y;
-    out.tex_coords = uv;
     return out;
 }
 
-// Fragment shader
  
-@group(0) @binding(0)
+
-var t_diffuse: texture_2d<f32>;
+
-@group(0) @binding(1)
+
-var s_diffuse: sampler;
+
+// @vertex
+// fn vs_main(@location(0) input: vec2f) -> VertexOutput {
+//     var out: VertexOutput;
+//     out.clip_position = vec4f(input, 0.0, 1.0);
+//     out.coords = input * 0.5 + vec2f(0.5, 0.5);
+//     return out;
+// }
+
+@group(0) @binding(0) var y_texture: texture_2d<f32>;
+@group(0) @binding(1) var uv_texture: texture_2d<f32>;
+@group(0) @binding(2) var texture_sampler: sampler;
+@group(0) @binding(3) var<uniform> rgb_primaries: mat3x3<f32>;
 
 @fragment
-fn fs_main(in: VertexOutput) -> @location(0) vec4<f32> {
+fn fs_main(input: VertexOutput) -> @location(0) vec4<f32> {
-    return textureSample(t_diffuse, s_diffuse, in.tex_coords);
+    let y = textureSample(y_texture,  texture_sampler, input.tex_coords).r;
+    let uv = textureSample(uv_texture, texture_sampler, input.tex_coords).rg;
+    let yuv = vec3f(y, uv);
+    let rgb = rgb_primaries * yuv;
+    return vec4f(rgb, 1.0);
 }
 

flake.nix

@@ -206,6 +206,7 @@
                     apple-sdk_26
                   ])
                   ++ (lib.optionals pkgs.stdenv.isLinux [
+                    ffmpeg
                     heaptrack
                     samply
                     cargo-flamegraph

justfile

@@ -12,6 +12,7 @@ hdrtest:
     GST_DEBUG=3 gst-launch-1.0 playbin3 uri=https://jellyfin.tsuba.darksailor.dev/Items/6010382cf25273e624d305907010d773/Download?api_key=036c140222464878862231ef66a2bc9c video-sink="videoconvert ! video/x-raw,format=(string)RGB10A2_LE ! fakesink"
     
 codec:
-    GST_DEBUG=3 gst-discoverer-1.0 -v https://jellyfin.tsuba.darksailor.dev/Items/6010382cf25273e624d305907010d773/Download?api_key=036c140222464878862231ef66a2bc9c
+    GST_DEBUG=3 gst-discoverer-1.0 https://jellyfin.tsuba.darksailor.dev/Items/6010382cf25273e624d305907010d773/Download?api_key=036c140222464878862231ef66a2bc9c
-    
+ffprobe:
+    ffprobe -v error -show_format -show_streams "https://jellyfin.tsuba.darksailor.dev/Items/6010382cf25273e624d305907010d773/Download?api_key=036c140222464878862231ef66a2bc9c" | grep pix_fmt