feat(iced-video): added video format to the video frame

crates/iced-video/src/primitive.rs

@@ -66,6 +66,14 @@ pub struct VideoFrame {
     pub size: wgpu::Extent3d,
     pub ready: Arc<AtomicBool>,
     pub frame: Arc<Mutex<gst::Sample>>,
+    pub format: VideoFormat,
+}
+
+#[derive(Debug, Clone, Copy)]
+pub enum ToneMapping {
+    None,
+    InverseOETF,
+    Reinhard,
 }
 
 impl iced_wgpu::Primitive for VideoFrame {
@@ -80,8 +88,13 @@ impl iced_wgpu::Primitive for VideoFrame {
         viewport: &iced_wgpu::graphics::Viewport,
     ) {
         let video = pipeline.videos.entry(self.id.clone()).or_insert_with(|| {
-            let texture =
-                VideoTexture::new("iced-video-texture", self.size, device, pipeline.format);
+            let texture = VideoTexture::new(
+                "iced-video-texture",
+                self.size,
+                device,
+                pipeline.format,
+                self.format,
+            );
             let conversion_matrix = if texture.format().is_wide() {
                 BT2020_TO_RGB
             } else {
@@ -235,12 +248,16 @@ impl iced_wgpu::Primitive for VideoFrame {
 
 /// NV12 or P010 are only supported in DX12 and Vulkan backends.
 /// While we can use vulkan with moltenvk on macos, I'd much rather use metal directly
+/// Right now only supports interleaved UV formats.
+/// For planar formats we would need 3 textures.
 #[derive(Debug)]
 pub struct VideoTexture {
     y: wgpu::Texture,
     uv: wgpu::Texture,
     size: wgpu::Extent3d,
-    pixel_format: gst::VideoFormat,
+    video_format: VideoFormat,
+    surface_format: wgpu::TextureFormat,
+    tone_mapping: ToneMapping,
 }
 
 impl VideoTexture {
@@ -252,9 +269,28 @@ impl VideoTexture {
         label: &str,
         size: wgpu::Extent3d,
         device: &wgpu::Device,
-        format: wgpu::TextureFormat,
+        surface_format: wgpu::TextureFormat,
         video_format: VideoFormat,
     ) -> Self {
+        let surface_hdr = surface_format.is_wide();
+        let video_hdr = matches!(video_format, VideoFormat::P01010le | VideoFormat::P016Le);
+
+        if surface_hdr && !video_hdr {
+            tracing::warn!("Surface texture is HDR but video format is SDR");
+        } else if !surface_hdr && video_hdr {
+            tracing::warn!("Video format is HDR but surface does not support HDR");
+        }
+
+        let tone_mapping = if surface_hdr && video_hdr {
+            ToneMapping::None
+        } else if surface_hdr && !video_hdr {
+            ToneMapping::InverseOETF
+        } else if !surface_hdr && video_hdr {
+            ToneMapping::Reinhard
+        } else {
+            ToneMapping::None
+        };
+
         let y_texture = device.create_texture(&wgpu::TextureDescriptor {
             label: Some(&format!("{}-y", label)),
             size: wgpu::Extent3d {
@@ -265,7 +301,7 @@ impl VideoTexture {
             mip_level_count: 1,
             sample_count: 1,
             dimension: wgpu::TextureDimension::D2,
-            format: wgpu::TextureFormat::R8Unorm,
+            format: wgpu::TextureFormat::R16Unorm,
             usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
             view_formats: &[],
         });
@@ -279,7 +315,7 @@ impl VideoTexture {
             mip_level_count: 1,
             sample_count: 1,
             dimension: wgpu::TextureDimension::D2,
-            format: wgpu::TextureFormat::Rg8Unorm,
+            format: wgpu::TextureFormat::Rg16Unorm,
             usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
             view_formats: &[],
         });
@@ -287,151 +323,81 @@ impl VideoTexture {
             y: y_texture,
             uv: uv_texture,
             size,
-            pixel_format: VideoFormat::Unknown,
+            surface_format,
+            video_format,
+            tone_mapping,
         }
     }
 
+    // This return the surface texture format, not the video pixel format
     pub fn format(&self) -> wgpu::TextureFormat {
-        match self {
-            VideoTexture::NV12(_) => wgpu::TextureFormat::NV12,
-            VideoTexture::P010(_) => wgpu::TextureFormat::P010,
-            VideoTexture::Composite { y, uv } => {
-                todo!()
-                // if y.format().is_wide() {
-                //     wgpu::TextureFormat::P010
-                // } else {
-                //     wgpu::TextureFormat::NV12
-                // }
-            }
-        }
+        self.surface_format
     }
 
     pub fn y_texture(&self) -> wgpu::TextureView {
-        match self {
-            VideoTexture::NV12(nv12) => nv12.create_view(&wgpu::TextureViewDescriptor {
-                label: Some("iced-video-texture-view-y-nv12"),
-                format: Some(wgpu::TextureFormat::R8Unorm),
-                ..Default::default()
-            }),
-            VideoTexture::P010(p010) => p010.create_view(&wgpu::TextureViewDescriptor {
-                label: Some("iced-video-texture-view-y-p010"),
-                format: Some(wgpu::TextureFormat::R16Unorm),
-                ..Default::default()
-            }),
-            VideoTexture::Composite { y, .. } => {
-                y.create_view(&wgpu::TextureViewDescriptor::default())
-            }
-        }
+        self.y.create_view(&wgpu::TextureViewDescriptor::default())
     }
 
     pub fn uv_texture(&self) -> wgpu::TextureView {
-        match self {
-            VideoTexture::NV12(nv12) => nv12.create_view(&wgpu::TextureViewDescriptor {
-                label: Some("iced-video-texture-view-uv-nv12"),
-                format: Some(wgpu::TextureFormat::Rg8Unorm),
-                ..Default::default()
-            }),
-            VideoTexture::P010(p010) => p010.create_view(&wgpu::TextureViewDescriptor {
-                label: Some("iced-video-texture-view-uv-p010"),
-                format: Some(wgpu::TextureFormat::Rg16Unorm),
-                ..Default::default()
-            }),
-            VideoTexture::Composite { uv, .. } => {
-                uv.create_view(&wgpu::TextureViewDescriptor::default())
-            }
-        }
+        self.uv.create_view(&wgpu::TextureViewDescriptor::default())
     }
 
     pub fn resize(&self, name: &str, new_size: wgpu::Extent3d, device: &wgpu::Device) -> Self {
-        VideoTexture::new(name, new_size, device, self.format())
+        VideoTexture::new(name, new_size, device, self.format(), self.video_format)
+    }
+
+    pub fn pixel_format(&self) -> VideoFormat {
+        self.video_format
+    }
+
+    pub fn set_pixel_format(&mut self, format: VideoFormat) {
+        self.video_format = format;
     }
 
     /// This assumes that the data is laid out correctly for the texture format.
     pub fn write_texture(&self, data: &[u8], queue: &wgpu::Queue) {
-        match self {
-            VideoTexture::NV12(nv12) => {
-                queue.write_texture(
-                    wgpu::TexelCopyTextureInfo {
-                        texture: nv12,
-                        mip_level: 0,
-                        origin: wgpu::Origin3d::ZERO,
-                        aspect: wgpu::TextureAspect::All,
-                    },
-                    data,
-                    wgpu::TexelCopyBufferLayout {
-                        offset: 0,
-                        bytes_per_row: Some(nv12.size().width * 3),
-                        rows_per_image: Some(nv12.size().height),
-                    },
-                    nv12.size(),
-                );
-            }
-            VideoTexture::P010(p010) => {
-                dbg!(&p010.size());
-                dbg!(data.len());
-                queue.write_texture(
-                    wgpu::TexelCopyTextureInfo {
-                        texture: p010,
-                        mip_level: 0,
-                        origin: wgpu::Origin3d::ZERO,
-                        aspect: wgpu::TextureAspect::All,
-                    },
-                    data,
-                    wgpu::TexelCopyBufferLayout {
-                        offset: 0,
-                        bytes_per_row: Some(p010.size().width * 3),
-                        rows_per_image: Some(p010.size().height),
-                    },
-                    p010.size(),
-                );
-            }
-            VideoTexture::Composite { y, uv } => {
-                let y_size = wgpu::Extent3d {
-                    width: y.size().width,
-                    height: y.size().height,
-                    depth_or_array_layers: 1,
-                };
-                let uv_size = wgpu::Extent3d {
-                    width: uv.size().width,
-                    height: uv.size().height,
-                    depth_or_array_layers: 1,
-                };
-                let y_data_size = (y_size.width * y_size.height) as usize;
-                let uv_data_size = (uv_size.width * uv_size.height * 2) as usize; // UV is interleaved
+        // let (y, u, v) = match self.video_format {
+        //     VideoFormat::Nv12 | VideoFormat::P01010le | VideoFormat::P016Le => (4, 1, 1),
+        //     _ => (1, 1),
+        // };
+        let Self { y, uv, .. } = self;
+        let y_size = y.size();
+        let uv_size = uv.size();
 
-                queue.write_texture(
-                    wgpu::TexelCopyTextureInfo {
-                        texture: y,
-                        mip_level: 0,
-                        origin: wgpu::Origin3d::ZERO,
-                        aspect: wgpu::TextureAspect::All,
-                    },
-                    &data[0..y_data_size],
-                    wgpu::TexelCopyBufferLayout {
-                        offset: 0,
-                        bytes_per_row: Some(y_size.width),
-                        rows_per_image: Some(y_size.height),
-                    },
-                    y_size,
-                );
+        let y_data_size = (y_size.width * y_size.height * 2) as usize;
+        let uv_data_size = (y_data_size / 2) as usize; // UV is interleaved
 
-                queue.write_texture(
-                    wgpu::TexelCopyTextureInfo {
-                        texture: uv,
-                        mip_level: 0,
-                        origin: wgpu::Origin3d::ZERO,
-                        aspect: wgpu::TextureAspect::All,
-                    },
-                    &data[y_data_size..(y_data_size + uv_data_size)],
-                    wgpu::TexelCopyBufferLayout {
-                        offset: 0,
-                        bytes_per_row: Some(uv_size.width * 2),
-                        rows_per_image: Some(uv_size.height),
-                    },
-                    uv_size,
-                );
-            }
-        }
+        queue.write_texture(
+            wgpu::TexelCopyTextureInfo {
+                texture: y,
+                mip_level: 0,
+                origin: wgpu::Origin3d::ZERO,
+                aspect: wgpu::TextureAspect::All,
+            },
+            &data[0..y_data_size],
+            wgpu::TexelCopyBufferLayout {
+                offset: 0,
+                bytes_per_row: Some(y_size.width),
+                rows_per_image: Some(y_size.height),
+            },
+            y_size,
+        );
+
+        queue.write_texture(
+            wgpu::TexelCopyTextureInfo {
+                texture: uv,
+                mip_level: 0,
+                origin: wgpu::Origin3d::ZERO,
+                aspect: wgpu::TextureAspect::All,
+            },
+            &data[y_data_size..(y_data_size + uv_data_size)],
+            wgpu::TexelCopyBufferLayout {
+                offset: 0,
+                bytes_per_row: Some(uv_size.width),
+                rows_per_image: Some(uv_size.height),
+            },
+            uv_size,
+        );
     }
 }
 
@@ -448,12 +414,6 @@ impl VideoFrameData {
     pub fn is_hdr(&self) -> bool {
         self.texture.format().is_wide()
     }
-    pub fn is_nv12(&self) -> bool {
-        matches!(self.texture.format(), wgpu::TextureFormat::NV12)
-    }
-    pub fn is_p010(&self) -> bool {
-        matches!(self.texture.format(), wgpu::TextureFormat::P010)
-    }
 }
 
 #[derive(Debug)]