From 97a7a632d4256c1776ac891953772b78e631759b Mon Sep 17 00:00:00 2001 From: uttarayan21 Date: Sun, 4 Jan 2026 23:02:47 +0530 Subject: [PATCH] feat(iced-video): implement planar YUV texture support with HDR conversion matrices and update dependencies --- Cargo.lock | 33 +- Cargo.toml | 1 + README.md | 44 ++ crates/iced-video/Cargo.toml | 2 + crates/iced-video/src/primitive.rs | 442 ++++++++++++++---- .../iced-video/src/shaders/passthrough.wgsl | 51 +- flake.nix | 1 + justfile | 5 +- 8 files changed, 469 insertions(+), 110 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index caf7d23..7039e4c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3437,7 +3437,7 @@ dependencies = [ [[package]] name = "iced" version = "0.14.0" -source = "git+https://github.com/uttarayan21/iced?branch=0.14#5846d52983d7e2eecc478130ba6373f0c1f82c94" +source = "git+https://github.com/uttarayan21/iced?branch=0.14#6fbe1ec83722c67cf7f43291254b357ffc5b6fb6" dependencies = [ "iced_core", "iced_debug 0.14.0 (git+https://github.com/uttarayan21/iced?branch=0.14)", @@ -3455,6 +3455,7 @@ dependencies = [ name = "iced-video" version = "0.1.0" dependencies = [ + "bytemuck", "error-stack", "futures-lite 2.6.1", "gst", @@ -3466,12 +3467,13 @@ dependencies = [ "thiserror 2.0.17", "tracing", "tracing-subscriber", + "wgpu", ] [[package]] name = "iced_beacon" version = "0.14.0" -source = "git+https://github.com/uttarayan21/iced?branch=0.14#5846d52983d7e2eecc478130ba6373f0c1f82c94" +source = "git+https://github.com/uttarayan21/iced?branch=0.14#6fbe1ec83722c67cf7f43291254b357ffc5b6fb6" dependencies = [ "bincode 1.3.3", "futures", @@ -3486,7 +3488,7 @@ dependencies = [ [[package]] name = "iced_core" version = "0.14.0" -source = "git+https://github.com/uttarayan21/iced?branch=0.14#5846d52983d7e2eecc478130ba6373f0c1f82c94" +source = "git+https://github.com/uttarayan21/iced?branch=0.14#6fbe1ec83722c67cf7f43291254b357ffc5b6fb6" dependencies = [ "bitflags 2.10.0", "bytes", @@ -3515,7 +3517,7 @@ dependencies = [ [[package]] name = "iced_debug" version = "0.14.0" -source = "git+https://github.com/uttarayan21/iced?branch=0.14#5846d52983d7e2eecc478130ba6373f0c1f82c94" +source = "git+https://github.com/uttarayan21/iced?branch=0.14#6fbe1ec83722c67cf7f43291254b357ffc5b6fb6" dependencies = [ "cargo-hot-protocol", "iced_beacon", @@ -3527,7 +3529,7 @@ dependencies = [ [[package]] name = "iced_devtools" version = "0.14.0" -source = "git+https://github.com/uttarayan21/iced?branch=0.14#5846d52983d7e2eecc478130ba6373f0c1f82c94" +source = "git+https://github.com/uttarayan21/iced?branch=0.14#6fbe1ec83722c67cf7f43291254b357ffc5b6fb6" dependencies = [ "iced_debug 0.14.0 (git+https://github.com/uttarayan21/iced?branch=0.14)", "iced_program 0.14.0 (git+https://github.com/uttarayan21/iced?branch=0.14)", @@ -3538,7 +3540,7 @@ dependencies = [ [[package]] name = "iced_futures" version = "0.14.0" -source = "git+https://github.com/uttarayan21/iced?branch=0.14#5846d52983d7e2eecc478130ba6373f0c1f82c94" +source = "git+https://github.com/uttarayan21/iced?branch=0.14#6fbe1ec83722c67cf7f43291254b357ffc5b6fb6" dependencies = [ "futures", "iced_core", @@ -3571,7 +3573,7 @@ dependencies = [ [[package]] name = "iced_graphics" version = "0.14.0" -source = "git+https://github.com/uttarayan21/iced?branch=0.14#5846d52983d7e2eecc478130ba6373f0c1f82c94" +source = "git+https://github.com/uttarayan21/iced?branch=0.14#6fbe1ec83722c67cf7f43291254b357ffc5b6fb6" dependencies = [ "bitflags 2.10.0", "bytemuck", @@ -3602,7 +3604,7 @@ dependencies = [ [[package]] name = "iced_program" version = "0.14.0" -source = "git+https://github.com/uttarayan21/iced?branch=0.14#5846d52983d7e2eecc478130ba6373f0c1f82c94" +source = "git+https://github.com/uttarayan21/iced?branch=0.14#6fbe1ec83722c67cf7f43291254b357ffc5b6fb6" dependencies = [ "iced_graphics 0.14.0 (git+https://github.com/uttarayan21/iced?branch=0.14)", "iced_runtime 0.14.0 (git+https://github.com/uttarayan21/iced?branch=0.14)", @@ -3611,7 +3613,7 @@ dependencies = [ [[package]] name = "iced_renderer" version = "0.14.0" -source = "git+https://github.com/uttarayan21/iced?branch=0.14#5846d52983d7e2eecc478130ba6373f0c1f82c94" +source = "git+https://github.com/uttarayan21/iced?branch=0.14#6fbe1ec83722c67cf7f43291254b357ffc5b6fb6" dependencies = [ "iced_graphics 0.14.0 (git+https://github.com/uttarayan21/iced?branch=0.14)", "iced_tiny_skia", @@ -3636,7 +3638,7 @@ dependencies = [ [[package]] name = "iced_runtime" version = "0.14.0" -source = "git+https://github.com/uttarayan21/iced?branch=0.14#5846d52983d7e2eecc478130ba6373f0c1f82c94" +source = "git+https://github.com/uttarayan21/iced?branch=0.14#6fbe1ec83722c67cf7f43291254b357ffc5b6fb6" dependencies = [ "bytes", "iced_core", @@ -3649,7 +3651,7 @@ dependencies = [ [[package]] name = "iced_tiny_skia" version = "0.14.0" -source = "git+https://github.com/uttarayan21/iced?branch=0.14#5846d52983d7e2eecc478130ba6373f0c1f82c94" +source = "git+https://github.com/uttarayan21/iced?branch=0.14#6fbe1ec83722c67cf7f43291254b357ffc5b6fb6" dependencies = [ "bytemuck", "cosmic-text 0.15.0", @@ -3665,7 +3667,7 @@ dependencies = [ [[package]] name = "iced_wgpu" version = "0.14.0" -source = "git+https://github.com/uttarayan21/iced?branch=0.14#5846d52983d7e2eecc478130ba6373f0c1f82c94" +source = "git+https://github.com/uttarayan21/iced?branch=0.14#6fbe1ec83722c67cf7f43291254b357ffc5b6fb6" dependencies = [ "bitflags 2.10.0", "bytemuck", @@ -3685,7 +3687,7 @@ dependencies = [ [[package]] name = "iced_widget" version = "0.14.2" -source = "git+https://github.com/uttarayan21/iced?branch=0.14#5846d52983d7e2eecc478130ba6373f0c1f82c94" +source = "git+https://github.com/uttarayan21/iced?branch=0.14#6fbe1ec83722c67cf7f43291254b357ffc5b6fb6" dependencies = [ "iced_renderer", "log", @@ -3716,7 +3718,7 @@ dependencies = [ [[package]] name = "iced_winit" version = "0.14.0" -source = "git+https://github.com/uttarayan21/iced?branch=0.14#5846d52983d7e2eecc478130ba6373f0c1f82c94" +source = "git+https://github.com/uttarayan21/iced?branch=0.14#6fbe1ec83722c67cf7f43291254b357ffc5b6fb6" dependencies = [ "iced_debug 0.14.0 (git+https://github.com/uttarayan21/iced?branch=0.14)", "iced_program 0.14.0 (git+https://github.com/uttarayan21/iced?branch=0.14)", @@ -4043,6 +4045,7 @@ name = "jello" version = "0.1.0" dependencies = [ "api", + "bytemuck", "clap", "clap-verbosity-flag", "clap_complete", @@ -8781,7 +8784,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 73eb071..a2fad21 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,6 +32,7 @@ license = "MIT" [dependencies] api = { version = "0.1.0", path = "api" } +bytemuck = { version = "1.24.0", features = ["derive"] } clap = { version = "4.5", features = ["derive"] } clap-verbosity-flag = { version = "3.0.4", features = ["tracing"] } clap_complete = "4.5" diff --git a/README.md b/README.md index bb8f736..4b34093 100644 --- a/README.md +++ b/README.md @@ -63,3 +63,47 @@ In the shader the components get uniformly normalized from [0..=1023] integer to Videos however are generally not stored in this format or any rgb format in general because it is not as efficient for (lossy) compression as YUV formats. Right now I don't want to deal with yuv formats so I'll use gstreamer caps to convert the video into `Rgba10a2` format + + +## Pixel formats and Planes +Dated: Sun Jan 4 09:09:16 AM IST 2026 +| value | count | quantile | percentage | frequency | +| --- | --- | --- | --- | --- | +| yuv420p | 1815 | 0.5067001675041876 | 50.67% | ************************************************** | +| yuv420p10le | 1572 | 0.4388609715242881 | 43.89% | ******************************************* | +| yuvj420p | 171 | 0.04773869346733668 | 4.77% | **** | +| rgba | 14 | 0.003908431044109436 | 0.39% | | +| yuvj444p | 10 | 0.0027917364600781687 | 0.28% | | + +For all of my media collection these are the pixel formats for all the videos + +### RGBA +Pretty self evident +8 channels for each of R, G, B and A +Hopefully shouldn't be too hard to make a function or possibly a lut that takes data from rgba and maps it to Rgb10a2Unorm + +```mermaid +packet ++8: "R" ++8: "G" ++8: "B" ++8: "A" +``` + + +### YUV +[All YUV formats](https://learn.microsoft.com/en-us/windows/win32/medfound/recommended-8-bit-yuv-formats-for-video-rendering#surface-definitions) +[10 and 16 bit yuv formats](https://learn.microsoft.com/en-us/windows/win32/medfound/10-bit-and-16-bit-yuv-video-formats) + +Y -> Luminance +U,V -> Chrominance + +p -> Planar +sp -> semi planar + +j -> full range + +planar formats have each of the channels in a contiguous array one after another +in semi-planar formats the y channel is seperate and uv channels are interleaved + + diff --git a/crates/iced-video/Cargo.toml b/crates/iced-video/Cargo.toml index 2e37a32..a98099f 100644 --- a/crates/iced-video/Cargo.toml +++ b/crates/iced-video/Cargo.toml @@ -4,6 +4,7 @@ version = "0.1.0" edition = "2024" [dependencies] +bytemuck = "1.24.0" error-stack = "0.6.0" futures-lite = "2.6.1" gst.workspace = true @@ -13,6 +14,7 @@ iced_renderer = { version = "0.14.0", features = ["iced_wgpu"] } iced_wgpu = { version = "0.14.0" } thiserror = "2.0.17" tracing = "0.1.43" +wgpu = { version = "27.0.1", features = ["vulkan"] } [dev-dependencies] iced.workspace = true diff --git a/crates/iced-video/src/primitive.rs b/crates/iced-video/src/primitive.rs index 70d01da..aa88dac 100644 --- a/crates/iced-video/src/primitive.rs +++ b/crates/iced-video/src/primitive.rs @@ -1,9 +1,65 @@ use crate::id; +use gst::videoconvertscale::VideoFormat; use iced_wgpu::primitive::Pipeline; use iced_wgpu::wgpu; use std::collections::BTreeMap; use std::sync::{Arc, Mutex, atomic::AtomicBool}; +#[derive(Clone, Copy, Debug, bytemuck::Zeroable, bytemuck::Pod)] +#[repr(transparent)] +pub struct ConversionMatrix { + matrix: [[f32; 4]; 4], +} + +// impl ConversionMatrix { +// pub fn desc() -> wgpu::VertexBufferLayout<'static> { +// wgpu::VertexBufferLayout { +// array_stride: core::mem::size_of::() as wgpu::BufferAddress, +// step_mode: wgpu::VertexStepMode::Vertex, +// attributes: &[ +// wgpu::VertexAttribute { +// offset: 0, +// shader_location: 0, +// format: wgpu::VertexFormat::Float32x4, +// }, +// wgpu::VertexAttribute { +// offset: 16, +// shader_location: 1, +// format: wgpu::VertexFormat::Float32x4, +// }, +// wgpu::VertexAttribute { +// offset: 32, +// shader_location: 2, +// format: wgpu::VertexFormat::Float32x4, +// }, +// wgpu::VertexAttribute { +// offset: 48, +// shader_location: 3, +// format: wgpu::VertexFormat::Float32x4, +// }, +// ], +// } +// } +// } + +pub const BT2020_TO_RGB: ConversionMatrix = ConversionMatrix { + matrix: [ + [1.1684, 0.0000, 1.6836, -0.9122], + [1.1684, -0.1873, -0.6520, 0.3015], + [1.1684, 2.1482, 0.0000, -1.1322], + [0.0, 0.0, 0.0, 1.0], + ], +}; + +pub const BT709_TO_RGB: ConversionMatrix = ConversionMatrix { + matrix: [ + [1.1644, 0.0000, 1.7927, -0.9729], + [1.1644, -0.2132, -0.5329, 0.3015], + [1.1644, 2.1124, 0.0000, -1.1334], + [0.0, 0.0, 0.0, 1.0], + ], +}; + #[derive(Debug)] pub struct VideoFrame { pub id: id::Id, @@ -24,73 +80,78 @@ impl iced_wgpu::Primitive for VideoFrame { viewport: &iced_wgpu::graphics::Viewport, ) { let video = pipeline.videos.entry(self.id.clone()).or_insert_with(|| { - let texture = device.create_texture(&wgpu::TextureDescriptor { - label: Some("iced-video-texture"), - size: self.size, - mip_level_count: 1, - sample_count: 1, - dimension: wgpu::TextureDimension::D2, - format: pipeline.format, - usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, - view_formats: &[], - }); + let texture = + VideoTexture::new("iced-video-texture", self.size, device, pipeline.format); + let conversion_matrix = if texture.format().is_wide() { + BT2020_TO_RGB + } else { + BT709_TO_RGB + }; let buffer = device.create_buffer(&wgpu::BufferDescriptor { - label: Some("iced-video-buffer"), - size: (self.size.width * self.size.height * 4) as u64, - usage: wgpu::BufferUsages::COPY_SRC | wgpu::BufferUsages::COPY_DST, + label: Some("iced-video-conversion-matrix-buffer"), + usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, + size: core::mem::size_of::() as wgpu::BufferAddress, mapped_at_creation: false, }); + let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor { label: Some("iced-video-texture-bind-group"), layout: &pipeline.bind_group_layout, entries: &[ wgpu::BindGroupEntry { binding: 0, - resource: wgpu::BindingResource::TextureView( - &texture.create_view(&wgpu::TextureViewDescriptor::default()), - ), + resource: wgpu::BindingResource::TextureView(&texture.y_texture()), }, wgpu::BindGroupEntry { binding: 1, + resource: wgpu::BindingResource::TextureView(&texture.uv_texture()), + }, + wgpu::BindGroupEntry { + binding: 2, resource: wgpu::BindingResource::Sampler(&pipeline.sampler), }, + wgpu::BindGroupEntry { + binding: 3, + resource: wgpu::BindingResource::Buffer(buffer.as_entire_buffer_binding()), + }, ], }); - VideoTextures { + + VideoFrameData { id: self.id.clone(), texture, - buffer, + conversion_matrix: buffer, bind_group, ready: Arc::clone(&self.ready), } }); - // dbg!(&self.size, video.texture.size()); if self.size != video.texture.size() { - // Resize the texture if the size has changed. - let new_texture = device.create_texture(&wgpu::TextureDescriptor { - label: Some("iced-video-texture-resized"), - size: self.size, - mip_level_count: 1, - sample_count: 1, - dimension: wgpu::TextureDimension::D2, - format: pipeline.format, - usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, - view_formats: &[], - }); + let new_texture = video + .texture + .resize("iced-video-texture-resized", self.size, device); + let new_bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor { - label: Some("iced-video-texture-bind-group-resized"), + label: Some("iced-video-texture-bind-group"), layout: &pipeline.bind_group_layout, entries: &[ wgpu::BindGroupEntry { binding: 0, - resource: wgpu::BindingResource::TextureView( - &new_texture.create_view(&wgpu::TextureViewDescriptor::default()), - ), + resource: wgpu::BindingResource::TextureView(&new_texture.y_texture()), }, wgpu::BindGroupEntry { binding: 1, + resource: wgpu::BindingResource::TextureView(&new_texture.uv_texture()), + }, + wgpu::BindGroupEntry { + binding: 2, resource: wgpu::BindingResource::Sampler(&pipeline.sampler), }, + wgpu::BindGroupEntry { + binding: 3, + resource: wgpu::BindingResource::Buffer( + video.conversion_matrix.as_entire_buffer_binding(), + ), + }, ], }); video.texture = new_texture; @@ -106,21 +167,24 @@ impl iced_wgpu::Primitive for VideoFrame { .map_readable() .expect("BUG: Failed to map gst::Buffer readable"); // queue.write_buffer(&video.buffer, 0, &data); - queue.write_texture( - wgpu::TexelCopyTextureInfo { - texture: &video.texture, - mip_level: 0, - origin: wgpu::Origin3d::ZERO, - aspect: wgpu::TextureAspect::All, - }, - &data, - wgpu::TexelCopyBufferLayout { - offset: 0, - bytes_per_row: Some(4 * self.size.width), - rows_per_image: Some(self.size.height), - }, - self.size, - ); + + video.texture.write_texture(&data, queue); + // queue.write_texture( + // wgpu::TexelCopyTextureInfo { + // texture: &video.texture, + // mip_level: 0, + // origin: wgpu::Origin3d::ZERO, + // aspect: wgpu::TextureAspect::All, + // }, + // &data, + // wgpu::TexelCopyBufferLayout { + // offset: 0, + // bytes_per_row: Some(4 * self.size.width), + // rows_per_image: Some(self.size.height), + // }, + // self.size, + // ); + drop(data); video .ready @@ -139,23 +203,6 @@ impl iced_wgpu::Primitive for VideoFrame { return; }; - // encoder.copy_buffer_to_texture( - // wgpu::TexelCopyBufferInfo { - // buffer: &video.buffer, - // layout: wgpu::TexelCopyBufferLayout { - // offset: 0, - // bytes_per_row: Some(4 * self.size.width), - // rows_per_image: Some(self.size.height), - // }, - // }, - // wgpu::TexelCopyTextureInfo { - // texture: &video.texture, - // mip_level: 0, - // origin: wgpu::Origin3d::ZERO, - // aspect: wgpu::TextureAspect::All, - // }, - // self.size, - // ); let mut render_pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { label: Some("iced-video-render-pass"), color_attachments: &[Some(wgpu::RenderPassColorAttachment { @@ -186,36 +233,251 @@ impl iced_wgpu::Primitive for VideoFrame { } } +/// NV12 or P010 are only supported in DX12 and Vulkan backends. +/// While we can use vulkan with moltenvk on macos, I'd much rather use metal directly #[derive(Debug)] -pub struct VideoTextures { +pub struct VideoTexture { + y: wgpu::Texture, + uv: wgpu::Texture, + size: wgpu::Extent3d, + pixel_format: gst::VideoFormat, +} + +impl VideoTexture { + pub fn size(&self) -> wgpu::Extent3d { + self.size + } + + pub fn new( + label: &str, + size: wgpu::Extent3d, + device: &wgpu::Device, + format: wgpu::TextureFormat, + video_format: VideoFormat, + ) -> Self { + let y_texture = device.create_texture(&wgpu::TextureDescriptor { + label: Some(&format!("{}-y", label)), + size: wgpu::Extent3d { + width: size.width, + height: size.height, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::R8Unorm, + usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, + view_formats: &[], + }); + let uv_texture = device.create_texture(&wgpu::TextureDescriptor { + label: Some(&format!("{}-uv", label)), + size: wgpu::Extent3d { + width: size.width / 2, + height: size.height / 2, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: wgpu::TextureFormat::Rg8Unorm, + usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, + view_formats: &[], + }); + VideoTexture { + y: y_texture, + uv: uv_texture, + size, + pixel_format: VideoFormat::Unknown, + } + } + + pub fn format(&self) -> wgpu::TextureFormat { + match self { + VideoTexture::NV12(_) => wgpu::TextureFormat::NV12, + VideoTexture::P010(_) => wgpu::TextureFormat::P010, + VideoTexture::Composite { y, uv } => { + todo!() + // if y.format().is_wide() { + // wgpu::TextureFormat::P010 + // } else { + // wgpu::TextureFormat::NV12 + // } + } + } + } + + pub fn y_texture(&self) -> wgpu::TextureView { + match self { + VideoTexture::NV12(nv12) => nv12.create_view(&wgpu::TextureViewDescriptor { + label: Some("iced-video-texture-view-y-nv12"), + format: Some(wgpu::TextureFormat::R8Unorm), + ..Default::default() + }), + VideoTexture::P010(p010) => p010.create_view(&wgpu::TextureViewDescriptor { + label: Some("iced-video-texture-view-y-p010"), + format: Some(wgpu::TextureFormat::R16Unorm), + ..Default::default() + }), + VideoTexture::Composite { y, .. } => { + y.create_view(&wgpu::TextureViewDescriptor::default()) + } + } + } + + pub fn uv_texture(&self) -> wgpu::TextureView { + match self { + VideoTexture::NV12(nv12) => nv12.create_view(&wgpu::TextureViewDescriptor { + label: Some("iced-video-texture-view-uv-nv12"), + format: Some(wgpu::TextureFormat::Rg8Unorm), + ..Default::default() + }), + VideoTexture::P010(p010) => p010.create_view(&wgpu::TextureViewDescriptor { + label: Some("iced-video-texture-view-uv-p010"), + format: Some(wgpu::TextureFormat::Rg16Unorm), + ..Default::default() + }), + VideoTexture::Composite { uv, .. } => { + uv.create_view(&wgpu::TextureViewDescriptor::default()) + } + } + } + + pub fn resize(&self, name: &str, new_size: wgpu::Extent3d, device: &wgpu::Device) -> Self { + VideoTexture::new(name, new_size, device, self.format()) + } + + /// This assumes that the data is laid out correctly for the texture format. + pub fn write_texture(&self, data: &[u8], queue: &wgpu::Queue) { + match self { + VideoTexture::NV12(nv12) => { + queue.write_texture( + wgpu::TexelCopyTextureInfo { + texture: nv12, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + data, + wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(nv12.size().width * 3), + rows_per_image: Some(nv12.size().height), + }, + nv12.size(), + ); + } + VideoTexture::P010(p010) => { + dbg!(&p010.size()); + dbg!(data.len()); + queue.write_texture( + wgpu::TexelCopyTextureInfo { + texture: p010, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + data, + wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(p010.size().width * 3), + rows_per_image: Some(p010.size().height), + }, + p010.size(), + ); + } + VideoTexture::Composite { y, uv } => { + let y_size = wgpu::Extent3d { + width: y.size().width, + height: y.size().height, + depth_or_array_layers: 1, + }; + let uv_size = wgpu::Extent3d { + width: uv.size().width, + height: uv.size().height, + depth_or_array_layers: 1, + }; + let y_data_size = (y_size.width * y_size.height) as usize; + let uv_data_size = (uv_size.width * uv_size.height * 2) as usize; // UV is interleaved + + queue.write_texture( + wgpu::TexelCopyTextureInfo { + texture: y, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + &data[0..y_data_size], + wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(y_size.width), + rows_per_image: Some(y_size.height), + }, + y_size, + ); + + queue.write_texture( + wgpu::TexelCopyTextureInfo { + texture: uv, + mip_level: 0, + origin: wgpu::Origin3d::ZERO, + aspect: wgpu::TextureAspect::All, + }, + &data[y_data_size..(y_data_size + uv_data_size)], + wgpu::TexelCopyBufferLayout { + offset: 0, + bytes_per_row: Some(uv_size.width * 2), + rows_per_image: Some(uv_size.height), + }, + uv_size, + ); + } + } + } +} + +#[derive(Debug)] +pub struct VideoFrameData { id: id::Id, - texture: wgpu::Texture, - buffer: wgpu::Buffer, + texture: VideoTexture, bind_group: wgpu::BindGroup, + conversion_matrix: wgpu::Buffer, ready: Arc, } +impl VideoFrameData { + pub fn is_hdr(&self) -> bool { + self.texture.format().is_wide() + } + pub fn is_nv12(&self) -> bool { + matches!(self.texture.format(), wgpu::TextureFormat::NV12) + } + pub fn is_p010(&self) -> bool { + matches!(self.texture.format(), wgpu::TextureFormat::P010) + } +} + #[derive(Debug)] pub struct VideoPipeline { pipeline: wgpu::RenderPipeline, bind_group_layout: wgpu::BindGroupLayout, sampler: wgpu::Sampler, - videos: BTreeMap, format: wgpu::TextureFormat, + videos: BTreeMap, } -pub trait HdrTextureFormatExt { - fn is_hdr(&self) -> bool; +pub trait WideTextureFormatExt { + fn is_wide(&self) -> bool; } -impl HdrTextureFormatExt for wgpu::TextureFormat { - fn is_hdr(&self) -> bool { +impl WideTextureFormatExt for wgpu::TextureFormat { + fn is_wide(&self) -> bool { matches!( self, wgpu::TextureFormat::Rgba16Float | wgpu::TextureFormat::Rgba32Float | wgpu::TextureFormat::Rgb10a2Unorm | wgpu::TextureFormat::Rgb10a2Uint + | wgpu::TextureFormat::P010 ) } } @@ -225,15 +487,14 @@ impl Pipeline for VideoPipeline { where Self: Sized, { - if format.is_hdr() { + if format.is_wide() { tracing::info!("HDR texture format detected: {:?}", format); } - let shader_passthrough = - device.create_shader_module(wgpu::include_wgsl!("shaders/passthrough.wgsl")); let bind_group_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { label: Some("iced-video-texture-bind-group-layout"), entries: &[ + // y wgpu::BindGroupLayoutEntry { binding: 0, visibility: wgpu::ShaderStages::FRAGMENT, @@ -244,15 +505,40 @@ impl Pipeline for VideoPipeline { }, count: None, }, + // uv wgpu::BindGroupLayoutEntry { binding: 1, visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + multisampled: false, + view_dimension: wgpu::TextureViewDimension::D2, + sample_type: wgpu::TextureSampleType::Float { filterable: true }, + }, + count: None, + }, + // sampler + wgpu::BindGroupLayoutEntry { + binding: 2, + visibility: wgpu::ShaderStages::FRAGMENT, ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), count: None, }, + // conversion matrix + wgpu::BindGroupLayoutEntry { + binding: 3, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Uniform, + has_dynamic_offset: false, + min_binding_size: None, + }, + count: None, + }, ], }); + let shader_passthrough = + device.create_shader_module(wgpu::include_wgsl!("shaders/passthrough.wgsl")); let render_pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { label: Some("iced-video-render-pipeline-layout"), @@ -273,7 +559,7 @@ impl Pipeline for VideoPipeline { entry_point: Some("fs_main"), targets: &[Some(wgpu::ColorTargetState { format, - blend: Some(wgpu::BlendState::ALPHA_BLENDING), + blend: Some(wgpu::BlendState::REPLACE), write_mask: wgpu::ColorWrites::ALL, })], compilation_options: wgpu::PipelineCompilationOptions::default(), diff --git a/crates/iced-video/src/shaders/passthrough.wgsl b/crates/iced-video/src/shaders/passthrough.wgsl index 1fd9706..db509db 100644 --- a/crates/iced-video/src/shaders/passthrough.wgsl +++ b/crates/iced-video/src/shaders/passthrough.wgsl @@ -1,31 +1,52 @@ -// Vertex shader +// struct VertexOutput { +// @builtin(position) clip_position: vec4f, +// @location(0) coords: vec2f, +// } + +// struct VertexInput { +// // @location(0) position: vec3, +// // @location(1) tex_coords: vec2, +// } struct VertexOutput { @builtin(position) clip_position: vec4, @location(0) tex_coords: vec2, -}; +} @vertex fn vs_main( - @builtin(vertex_index) in_vertex_index: u32, -) -> VertexOutput { + // model: VertexInput, +) -> VertexOutput { var out: VertexOutput; - let uv = vec2(f32((in_vertex_index << 1u) & 2u), f32(in_vertex_index & 2u)); - out.clip_position = vec4(uv * 2.0 - 1.0, 0.0, 1.0); - out.clip_position.y = -out.clip_position.y; - out.tex_coords = uv; + out.tex_coords = vec2(0.0, 0.0); + out.clip_position = vec4(0,0,0, 1.0); return out; } -// Fragment shader + -@group(0) @binding(0) -var t_diffuse: texture_2d; -@group(0) @binding(1) -var s_diffuse: sampler; + + + +// @vertex +// fn vs_main(@location(0) input: vec2f) -> VertexOutput { +// var out: VertexOutput; +// out.clip_position = vec4f(input, 0.0, 1.0); +// out.coords = input * 0.5 + vec2f(0.5, 0.5); +// return out; +// } + +@group(0) @binding(0) var y_texture: texture_2d; +@group(0) @binding(1) var uv_texture: texture_2d; +@group(0) @binding(2) var texture_sampler: sampler; +@group(0) @binding(3) var rgb_primaries: mat3x3; @fragment -fn fs_main(in: VertexOutput) -> @location(0) vec4 { - return textureSample(t_diffuse, s_diffuse, in.tex_coords); +fn fs_main(input: VertexOutput) -> @location(0) vec4 { + let y = textureSample(y_texture, texture_sampler, input.tex_coords).r; + let uv = textureSample(uv_texture, texture_sampler, input.tex_coords).rg; + let yuv = vec3f(y, uv); + let rgb = rgb_primaries * yuv; + return vec4f(rgb, 1.0); } diff --git a/flake.nix b/flake.nix index 0764296..c372d10 100644 --- a/flake.nix +++ b/flake.nix @@ -206,6 +206,7 @@ apple-sdk_26 ]) ++ (lib.optionals pkgs.stdenv.isLinux [ + ffmpeg heaptrack samply cargo-flamegraph diff --git a/justfile b/justfile index 19eb4cc..4c2dd88 100644 --- a/justfile +++ b/justfile @@ -12,6 +12,7 @@ hdrtest: GST_DEBUG=3 gst-launch-1.0 playbin3 uri=https://jellyfin.tsuba.darksailor.dev/Items/6010382cf25273e624d305907010d773/Download?api_key=036c140222464878862231ef66a2bc9c video-sink="videoconvert ! video/x-raw,format=(string)RGB10A2_LE ! fakesink" codec: - GST_DEBUG=3 gst-discoverer-1.0 -v https://jellyfin.tsuba.darksailor.dev/Items/6010382cf25273e624d305907010d773/Download?api_key=036c140222464878862231ef66a2bc9c - + GST_DEBUG=3 gst-discoverer-1.0 https://jellyfin.tsuba.darksailor.dev/Items/6010382cf25273e624d305907010d773/Download?api_key=036c140222464878862231ef66a2bc9c +ffprobe: + ffprobe -v error -show_format -show_streams "https://jellyfin.tsuba.darksailor.dev/Items/6010382cf25273e624d305907010d773/Download?api_key=036c140222464878862231ef66a2bc9c" | grep pix_fmt