Compare commits

..

3 Commits

Author SHA1 Message Date
uttarayan21
97a7a632d4 feat(iced-video): implement planar YUV texture support with HDR conversion matrices and update dependencies
Some checks failed
build / checks-matrix (push) Has been cancelled
build / codecov (push) Has been cancelled
docs / docs (push) Has been cancelled
build / checks-build (push) Has been cancelled
2026-01-04 23:02:47 +05:30
uttarayan21
29390140cd feat(settings): simplify form updates and temporarily disable server toggler 2025-12-27 00:13:54 +05:30
uttarayan21
97c2b3f14c feat(settings): implement user and server form handling with update functions and UI views 2025-12-27 00:04:42 +05:30
9 changed files with 600 additions and 120 deletions

33
Cargo.lock generated
View File

@@ -3437,7 +3437,7 @@ dependencies = [
[[package]]
name = "iced"
version = "0.14.0"
source = "git+https://github.com/uttarayan21/iced?branch=0.14#5846d52983d7e2eecc478130ba6373f0c1f82c94"
source = "git+https://github.com/uttarayan21/iced?branch=0.14#6fbe1ec83722c67cf7f43291254b357ffc5b6fb6"
dependencies = [
"iced_core",
"iced_debug 0.14.0 (git+https://github.com/uttarayan21/iced?branch=0.14)",
@@ -3455,6 +3455,7 @@ dependencies = [
name = "iced-video"
version = "0.1.0"
dependencies = [
"bytemuck",
"error-stack",
"futures-lite 2.6.1",
"gst",
@@ -3466,12 +3467,13 @@ dependencies = [
"thiserror 2.0.17",
"tracing",
"tracing-subscriber",
"wgpu",
]
[[package]]
name = "iced_beacon"
version = "0.14.0"
source = "git+https://github.com/uttarayan21/iced?branch=0.14#5846d52983d7e2eecc478130ba6373f0c1f82c94"
source = "git+https://github.com/uttarayan21/iced?branch=0.14#6fbe1ec83722c67cf7f43291254b357ffc5b6fb6"
dependencies = [
"bincode 1.3.3",
"futures",
@@ -3486,7 +3488,7 @@ dependencies = [
[[package]]
name = "iced_core"
version = "0.14.0"
source = "git+https://github.com/uttarayan21/iced?branch=0.14#5846d52983d7e2eecc478130ba6373f0c1f82c94"
source = "git+https://github.com/uttarayan21/iced?branch=0.14#6fbe1ec83722c67cf7f43291254b357ffc5b6fb6"
dependencies = [
"bitflags 2.10.0",
"bytes",
@@ -3515,7 +3517,7 @@ dependencies = [
[[package]]
name = "iced_debug"
version = "0.14.0"
source = "git+https://github.com/uttarayan21/iced?branch=0.14#5846d52983d7e2eecc478130ba6373f0c1f82c94"
source = "git+https://github.com/uttarayan21/iced?branch=0.14#6fbe1ec83722c67cf7f43291254b357ffc5b6fb6"
dependencies = [
"cargo-hot-protocol",
"iced_beacon",
@@ -3527,7 +3529,7 @@ dependencies = [
[[package]]
name = "iced_devtools"
version = "0.14.0"
source = "git+https://github.com/uttarayan21/iced?branch=0.14#5846d52983d7e2eecc478130ba6373f0c1f82c94"
source = "git+https://github.com/uttarayan21/iced?branch=0.14#6fbe1ec83722c67cf7f43291254b357ffc5b6fb6"
dependencies = [
"iced_debug 0.14.0 (git+https://github.com/uttarayan21/iced?branch=0.14)",
"iced_program 0.14.0 (git+https://github.com/uttarayan21/iced?branch=0.14)",
@@ -3538,7 +3540,7 @@ dependencies = [
[[package]]
name = "iced_futures"
version = "0.14.0"
source = "git+https://github.com/uttarayan21/iced?branch=0.14#5846d52983d7e2eecc478130ba6373f0c1f82c94"
source = "git+https://github.com/uttarayan21/iced?branch=0.14#6fbe1ec83722c67cf7f43291254b357ffc5b6fb6"
dependencies = [
"futures",
"iced_core",
@@ -3571,7 +3573,7 @@ dependencies = [
[[package]]
name = "iced_graphics"
version = "0.14.0"
source = "git+https://github.com/uttarayan21/iced?branch=0.14#5846d52983d7e2eecc478130ba6373f0c1f82c94"
source = "git+https://github.com/uttarayan21/iced?branch=0.14#6fbe1ec83722c67cf7f43291254b357ffc5b6fb6"
dependencies = [
"bitflags 2.10.0",
"bytemuck",
@@ -3602,7 +3604,7 @@ dependencies = [
[[package]]
name = "iced_program"
version = "0.14.0"
source = "git+https://github.com/uttarayan21/iced?branch=0.14#5846d52983d7e2eecc478130ba6373f0c1f82c94"
source = "git+https://github.com/uttarayan21/iced?branch=0.14#6fbe1ec83722c67cf7f43291254b357ffc5b6fb6"
dependencies = [
"iced_graphics 0.14.0 (git+https://github.com/uttarayan21/iced?branch=0.14)",
"iced_runtime 0.14.0 (git+https://github.com/uttarayan21/iced?branch=0.14)",
@@ -3611,7 +3613,7 @@ dependencies = [
[[package]]
name = "iced_renderer"
version = "0.14.0"
source = "git+https://github.com/uttarayan21/iced?branch=0.14#5846d52983d7e2eecc478130ba6373f0c1f82c94"
source = "git+https://github.com/uttarayan21/iced?branch=0.14#6fbe1ec83722c67cf7f43291254b357ffc5b6fb6"
dependencies = [
"iced_graphics 0.14.0 (git+https://github.com/uttarayan21/iced?branch=0.14)",
"iced_tiny_skia",
@@ -3636,7 +3638,7 @@ dependencies = [
[[package]]
name = "iced_runtime"
version = "0.14.0"
source = "git+https://github.com/uttarayan21/iced?branch=0.14#5846d52983d7e2eecc478130ba6373f0c1f82c94"
source = "git+https://github.com/uttarayan21/iced?branch=0.14#6fbe1ec83722c67cf7f43291254b357ffc5b6fb6"
dependencies = [
"bytes",
"iced_core",
@@ -3649,7 +3651,7 @@ dependencies = [
[[package]]
name = "iced_tiny_skia"
version = "0.14.0"
source = "git+https://github.com/uttarayan21/iced?branch=0.14#5846d52983d7e2eecc478130ba6373f0c1f82c94"
source = "git+https://github.com/uttarayan21/iced?branch=0.14#6fbe1ec83722c67cf7f43291254b357ffc5b6fb6"
dependencies = [
"bytemuck",
"cosmic-text 0.15.0",
@@ -3665,7 +3667,7 @@ dependencies = [
[[package]]
name = "iced_wgpu"
version = "0.14.0"
source = "git+https://github.com/uttarayan21/iced?branch=0.14#5846d52983d7e2eecc478130ba6373f0c1f82c94"
source = "git+https://github.com/uttarayan21/iced?branch=0.14#6fbe1ec83722c67cf7f43291254b357ffc5b6fb6"
dependencies = [
"bitflags 2.10.0",
"bytemuck",
@@ -3685,7 +3687,7 @@ dependencies = [
[[package]]
name = "iced_widget"
version = "0.14.2"
source = "git+https://github.com/uttarayan21/iced?branch=0.14#5846d52983d7e2eecc478130ba6373f0c1f82c94"
source = "git+https://github.com/uttarayan21/iced?branch=0.14#6fbe1ec83722c67cf7f43291254b357ffc5b6fb6"
dependencies = [
"iced_renderer",
"log",
@@ -3716,7 +3718,7 @@ dependencies = [
[[package]]
name = "iced_winit"
version = "0.14.0"
source = "git+https://github.com/uttarayan21/iced?branch=0.14#5846d52983d7e2eecc478130ba6373f0c1f82c94"
source = "git+https://github.com/uttarayan21/iced?branch=0.14#6fbe1ec83722c67cf7f43291254b357ffc5b6fb6"
dependencies = [
"iced_debug 0.14.0 (git+https://github.com/uttarayan21/iced?branch=0.14)",
"iced_program 0.14.0 (git+https://github.com/uttarayan21/iced?branch=0.14)",
@@ -4043,6 +4045,7 @@ name = "jello"
version = "0.1.0"
dependencies = [
"api",
"bytemuck",
"clap",
"clap-verbosity-flag",
"clap_complete",
@@ -8781,7 +8784,7 @@ version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
dependencies = [
"windows-sys 0.61.2",
"windows-sys 0.48.0",
]
[[package]]

View File

@@ -32,6 +32,7 @@ license = "MIT"
[dependencies]
api = { version = "0.1.0", path = "api" }
bytemuck = { version = "1.24.0", features = ["derive"] }
clap = { version = "4.5", features = ["derive"] }
clap-verbosity-flag = { version = "3.0.4", features = ["tracing"] }
clap_complete = "4.5"

View File

@@ -63,3 +63,47 @@ In the shader the components get uniformly normalized from [0..=1023] integer to
Videos however are generally not stored in this format or any rgb format in general because it is not as efficient for (lossy) compression as YUV formats.
Right now I don't want to deal with yuv formats so I'll use gstreamer caps to convert the video into `Rgba10a2` format
## Pixel formats and Planes
Dated: Sun Jan 4 09:09:16 AM IST 2026
| value | count | quantile | percentage | frequency |
| --- | --- | --- | --- | --- |
| yuv420p | 1815 | 0.5067001675041876 | 50.67% | ************************************************** |
| yuv420p10le | 1572 | 0.4388609715242881 | 43.89% | ******************************************* |
| yuvj420p | 171 | 0.04773869346733668 | 4.77% | **** |
| rgba | 14 | 0.003908431044109436 | 0.39% | |
| yuvj444p | 10 | 0.0027917364600781687 | 0.28% | |
For all of my media collection these are the pixel formats for all the videos
### RGBA
Pretty self evident
8 channels for each of R, G, B and A
Hopefully shouldn't be too hard to make a function or possibly a lut that takes data from rgba and maps it to Rgb10a2Unorm
```mermaid
packet
+8: "R"
+8: "G"
+8: "B"
+8: "A"
```
### YUV
[All YUV formats](https://learn.microsoft.com/en-us/windows/win32/medfound/recommended-8-bit-yuv-formats-for-video-rendering#surface-definitions)
[10 and 16 bit yuv formats](https://learn.microsoft.com/en-us/windows/win32/medfound/10-bit-and-16-bit-yuv-video-formats)
Y -> Luminance
U,V -> Chrominance
p -> Planar
sp -> semi planar
j -> full range
planar formats have each of the channels in a contiguous array one after another
in semi-planar formats the y channel is seperate and uv channels are interleaved

View File

@@ -4,6 +4,7 @@ version = "0.1.0"
edition = "2024"
[dependencies]
bytemuck = "1.24.0"
error-stack = "0.6.0"
futures-lite = "2.6.1"
gst.workspace = true
@@ -13,6 +14,7 @@ iced_renderer = { version = "0.14.0", features = ["iced_wgpu"] }
iced_wgpu = { version = "0.14.0" }
thiserror = "2.0.17"
tracing = "0.1.43"
wgpu = { version = "27.0.1", features = ["vulkan"] }
[dev-dependencies]
iced.workspace = true

View File

@@ -1,9 +1,65 @@
use crate::id;
use gst::videoconvertscale::VideoFormat;
use iced_wgpu::primitive::Pipeline;
use iced_wgpu::wgpu;
use std::collections::BTreeMap;
use std::sync::{Arc, Mutex, atomic::AtomicBool};
#[derive(Clone, Copy, Debug, bytemuck::Zeroable, bytemuck::Pod)]
#[repr(transparent)]
pub struct ConversionMatrix {
matrix: [[f32; 4]; 4],
}
// impl ConversionMatrix {
// pub fn desc() -> wgpu::VertexBufferLayout<'static> {
// wgpu::VertexBufferLayout {
// array_stride: core::mem::size_of::<ConversionMatrix>() as wgpu::BufferAddress,
// step_mode: wgpu::VertexStepMode::Vertex,
// attributes: &[
// wgpu::VertexAttribute {
// offset: 0,
// shader_location: 0,
// format: wgpu::VertexFormat::Float32x4,
// },
// wgpu::VertexAttribute {
// offset: 16,
// shader_location: 1,
// format: wgpu::VertexFormat::Float32x4,
// },
// wgpu::VertexAttribute {
// offset: 32,
// shader_location: 2,
// format: wgpu::VertexFormat::Float32x4,
// },
// wgpu::VertexAttribute {
// offset: 48,
// shader_location: 3,
// format: wgpu::VertexFormat::Float32x4,
// },
// ],
// }
// }
// }
pub const BT2020_TO_RGB: ConversionMatrix = ConversionMatrix {
matrix: [
[1.1684, 0.0000, 1.6836, -0.9122],
[1.1684, -0.1873, -0.6520, 0.3015],
[1.1684, 2.1482, 0.0000, -1.1322],
[0.0, 0.0, 0.0, 1.0],
],
};
pub const BT709_TO_RGB: ConversionMatrix = ConversionMatrix {
matrix: [
[1.1644, 0.0000, 1.7927, -0.9729],
[1.1644, -0.2132, -0.5329, 0.3015],
[1.1644, 2.1124, 0.0000, -1.1334],
[0.0, 0.0, 0.0, 1.0],
],
};
#[derive(Debug)]
pub struct VideoFrame {
pub id: id::Id,
@@ -24,73 +80,78 @@ impl iced_wgpu::Primitive for VideoFrame {
viewport: &iced_wgpu::graphics::Viewport,
) {
let video = pipeline.videos.entry(self.id.clone()).or_insert_with(|| {
let texture = device.create_texture(&wgpu::TextureDescriptor {
label: Some("iced-video-texture"),
size: self.size,
mip_level_count: 1,
sample_count: 1,
dimension: wgpu::TextureDimension::D2,
format: pipeline.format,
usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
view_formats: &[],
});
let texture =
VideoTexture::new("iced-video-texture", self.size, device, pipeline.format);
let conversion_matrix = if texture.format().is_wide() {
BT2020_TO_RGB
} else {
BT709_TO_RGB
};
let buffer = device.create_buffer(&wgpu::BufferDescriptor {
label: Some("iced-video-buffer"),
size: (self.size.width * self.size.height * 4) as u64,
usage: wgpu::BufferUsages::COPY_SRC | wgpu::BufferUsages::COPY_DST,
label: Some("iced-video-conversion-matrix-buffer"),
usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST,
size: core::mem::size_of::<ConversionMatrix>() as wgpu::BufferAddress,
mapped_at_creation: false,
});
let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor {
label: Some("iced-video-texture-bind-group"),
layout: &pipeline.bind_group_layout,
entries: &[
wgpu::BindGroupEntry {
binding: 0,
resource: wgpu::BindingResource::TextureView(
&texture.create_view(&wgpu::TextureViewDescriptor::default()),
),
resource: wgpu::BindingResource::TextureView(&texture.y_texture()),
},
wgpu::BindGroupEntry {
binding: 1,
resource: wgpu::BindingResource::TextureView(&texture.uv_texture()),
},
wgpu::BindGroupEntry {
binding: 2,
resource: wgpu::BindingResource::Sampler(&pipeline.sampler),
},
wgpu::BindGroupEntry {
binding: 3,
resource: wgpu::BindingResource::Buffer(buffer.as_entire_buffer_binding()),
},
],
});
VideoTextures {
VideoFrameData {
id: self.id.clone(),
texture,
buffer,
conversion_matrix: buffer,
bind_group,
ready: Arc::clone(&self.ready),
}
});
// dbg!(&self.size, video.texture.size());
if self.size != video.texture.size() {
// Resize the texture if the size has changed.
let new_texture = device.create_texture(&wgpu::TextureDescriptor {
label: Some("iced-video-texture-resized"),
size: self.size,
mip_level_count: 1,
sample_count: 1,
dimension: wgpu::TextureDimension::D2,
format: pipeline.format,
usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
view_formats: &[],
});
let new_texture = video
.texture
.resize("iced-video-texture-resized", self.size, device);
let new_bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor {
label: Some("iced-video-texture-bind-group-resized"),
label: Some("iced-video-texture-bind-group"),
layout: &pipeline.bind_group_layout,
entries: &[
wgpu::BindGroupEntry {
binding: 0,
resource: wgpu::BindingResource::TextureView(
&new_texture.create_view(&wgpu::TextureViewDescriptor::default()),
),
resource: wgpu::BindingResource::TextureView(&new_texture.y_texture()),
},
wgpu::BindGroupEntry {
binding: 1,
resource: wgpu::BindingResource::TextureView(&new_texture.uv_texture()),
},
wgpu::BindGroupEntry {
binding: 2,
resource: wgpu::BindingResource::Sampler(&pipeline.sampler),
},
wgpu::BindGroupEntry {
binding: 3,
resource: wgpu::BindingResource::Buffer(
video.conversion_matrix.as_entire_buffer_binding(),
),
},
],
});
video.texture = new_texture;
@@ -106,21 +167,24 @@ impl iced_wgpu::Primitive for VideoFrame {
.map_readable()
.expect("BUG: Failed to map gst::Buffer readable");
// queue.write_buffer(&video.buffer, 0, &data);
queue.write_texture(
wgpu::TexelCopyTextureInfo {
texture: &video.texture,
mip_level: 0,
origin: wgpu::Origin3d::ZERO,
aspect: wgpu::TextureAspect::All,
},
&data,
wgpu::TexelCopyBufferLayout {
offset: 0,
bytes_per_row: Some(4 * self.size.width),
rows_per_image: Some(self.size.height),
},
self.size,
);
video.texture.write_texture(&data, queue);
// queue.write_texture(
// wgpu::TexelCopyTextureInfo {
// texture: &video.texture,
// mip_level: 0,
// origin: wgpu::Origin3d::ZERO,
// aspect: wgpu::TextureAspect::All,
// },
// &data,
// wgpu::TexelCopyBufferLayout {
// offset: 0,
// bytes_per_row: Some(4 * self.size.width),
// rows_per_image: Some(self.size.height),
// },
// self.size,
// );
drop(data);
video
.ready
@@ -139,23 +203,6 @@ impl iced_wgpu::Primitive for VideoFrame {
return;
};
// encoder.copy_buffer_to_texture(
// wgpu::TexelCopyBufferInfo {
// buffer: &video.buffer,
// layout: wgpu::TexelCopyBufferLayout {
// offset: 0,
// bytes_per_row: Some(4 * self.size.width),
// rows_per_image: Some(self.size.height),
// },
// },
// wgpu::TexelCopyTextureInfo {
// texture: &video.texture,
// mip_level: 0,
// origin: wgpu::Origin3d::ZERO,
// aspect: wgpu::TextureAspect::All,
// },
// self.size,
// );
let mut render_pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor {
label: Some("iced-video-render-pass"),
color_attachments: &[Some(wgpu::RenderPassColorAttachment {
@@ -186,36 +233,251 @@ impl iced_wgpu::Primitive for VideoFrame {
}
}
/// NV12 or P010 are only supported in DX12 and Vulkan backends.
/// While we can use vulkan with moltenvk on macos, I'd much rather use metal directly
#[derive(Debug)]
pub struct VideoTextures {
pub struct VideoTexture {
y: wgpu::Texture,
uv: wgpu::Texture,
size: wgpu::Extent3d,
pixel_format: gst::VideoFormat,
}
impl VideoTexture {
pub fn size(&self) -> wgpu::Extent3d {
self.size
}
pub fn new(
label: &str,
size: wgpu::Extent3d,
device: &wgpu::Device,
format: wgpu::TextureFormat,
video_format: VideoFormat,
) -> Self {
let y_texture = device.create_texture(&wgpu::TextureDescriptor {
label: Some(&format!("{}-y", label)),
size: wgpu::Extent3d {
width: size.width,
height: size.height,
depth_or_array_layers: 1,
},
mip_level_count: 1,
sample_count: 1,
dimension: wgpu::TextureDimension::D2,
format: wgpu::TextureFormat::R8Unorm,
usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
view_formats: &[],
});
let uv_texture = device.create_texture(&wgpu::TextureDescriptor {
label: Some(&format!("{}-uv", label)),
size: wgpu::Extent3d {
width: size.width / 2,
height: size.height / 2,
depth_or_array_layers: 1,
},
mip_level_count: 1,
sample_count: 1,
dimension: wgpu::TextureDimension::D2,
format: wgpu::TextureFormat::Rg8Unorm,
usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
view_formats: &[],
});
VideoTexture {
y: y_texture,
uv: uv_texture,
size,
pixel_format: VideoFormat::Unknown,
}
}
pub fn format(&self) -> wgpu::TextureFormat {
match self {
VideoTexture::NV12(_) => wgpu::TextureFormat::NV12,
VideoTexture::P010(_) => wgpu::TextureFormat::P010,
VideoTexture::Composite { y, uv } => {
todo!()
// if y.format().is_wide() {
// wgpu::TextureFormat::P010
// } else {
// wgpu::TextureFormat::NV12
// }
}
}
}
pub fn y_texture(&self) -> wgpu::TextureView {
match self {
VideoTexture::NV12(nv12) => nv12.create_view(&wgpu::TextureViewDescriptor {
label: Some("iced-video-texture-view-y-nv12"),
format: Some(wgpu::TextureFormat::R8Unorm),
..Default::default()
}),
VideoTexture::P010(p010) => p010.create_view(&wgpu::TextureViewDescriptor {
label: Some("iced-video-texture-view-y-p010"),
format: Some(wgpu::TextureFormat::R16Unorm),
..Default::default()
}),
VideoTexture::Composite { y, .. } => {
y.create_view(&wgpu::TextureViewDescriptor::default())
}
}
}
pub fn uv_texture(&self) -> wgpu::TextureView {
match self {
VideoTexture::NV12(nv12) => nv12.create_view(&wgpu::TextureViewDescriptor {
label: Some("iced-video-texture-view-uv-nv12"),
format: Some(wgpu::TextureFormat::Rg8Unorm),
..Default::default()
}),
VideoTexture::P010(p010) => p010.create_view(&wgpu::TextureViewDescriptor {
label: Some("iced-video-texture-view-uv-p010"),
format: Some(wgpu::TextureFormat::Rg16Unorm),
..Default::default()
}),
VideoTexture::Composite { uv, .. } => {
uv.create_view(&wgpu::TextureViewDescriptor::default())
}
}
}
pub fn resize(&self, name: &str, new_size: wgpu::Extent3d, device: &wgpu::Device) -> Self {
VideoTexture::new(name, new_size, device, self.format())
}
/// This assumes that the data is laid out correctly for the texture format.
pub fn write_texture(&self, data: &[u8], queue: &wgpu::Queue) {
match self {
VideoTexture::NV12(nv12) => {
queue.write_texture(
wgpu::TexelCopyTextureInfo {
texture: nv12,
mip_level: 0,
origin: wgpu::Origin3d::ZERO,
aspect: wgpu::TextureAspect::All,
},
data,
wgpu::TexelCopyBufferLayout {
offset: 0,
bytes_per_row: Some(nv12.size().width * 3),
rows_per_image: Some(nv12.size().height),
},
nv12.size(),
);
}
VideoTexture::P010(p010) => {
dbg!(&p010.size());
dbg!(data.len());
queue.write_texture(
wgpu::TexelCopyTextureInfo {
texture: p010,
mip_level: 0,
origin: wgpu::Origin3d::ZERO,
aspect: wgpu::TextureAspect::All,
},
data,
wgpu::TexelCopyBufferLayout {
offset: 0,
bytes_per_row: Some(p010.size().width * 3),
rows_per_image: Some(p010.size().height),
},
p010.size(),
);
}
VideoTexture::Composite { y, uv } => {
let y_size = wgpu::Extent3d {
width: y.size().width,
height: y.size().height,
depth_or_array_layers: 1,
};
let uv_size = wgpu::Extent3d {
width: uv.size().width,
height: uv.size().height,
depth_or_array_layers: 1,
};
let y_data_size = (y_size.width * y_size.height) as usize;
let uv_data_size = (uv_size.width * uv_size.height * 2) as usize; // UV is interleaved
queue.write_texture(
wgpu::TexelCopyTextureInfo {
texture: y,
mip_level: 0,
origin: wgpu::Origin3d::ZERO,
aspect: wgpu::TextureAspect::All,
},
&data[0..y_data_size],
wgpu::TexelCopyBufferLayout {
offset: 0,
bytes_per_row: Some(y_size.width),
rows_per_image: Some(y_size.height),
},
y_size,
);
queue.write_texture(
wgpu::TexelCopyTextureInfo {
texture: uv,
mip_level: 0,
origin: wgpu::Origin3d::ZERO,
aspect: wgpu::TextureAspect::All,
},
&data[y_data_size..(y_data_size + uv_data_size)],
wgpu::TexelCopyBufferLayout {
offset: 0,
bytes_per_row: Some(uv_size.width * 2),
rows_per_image: Some(uv_size.height),
},
uv_size,
);
}
}
}
}
#[derive(Debug)]
pub struct VideoFrameData {
id: id::Id,
texture: wgpu::Texture,
buffer: wgpu::Buffer,
texture: VideoTexture,
bind_group: wgpu::BindGroup,
conversion_matrix: wgpu::Buffer,
ready: Arc<AtomicBool>,
}
impl VideoFrameData {
pub fn is_hdr(&self) -> bool {
self.texture.format().is_wide()
}
pub fn is_nv12(&self) -> bool {
matches!(self.texture.format(), wgpu::TextureFormat::NV12)
}
pub fn is_p010(&self) -> bool {
matches!(self.texture.format(), wgpu::TextureFormat::P010)
}
}
#[derive(Debug)]
pub struct VideoPipeline {
pipeline: wgpu::RenderPipeline,
bind_group_layout: wgpu::BindGroupLayout,
sampler: wgpu::Sampler,
videos: BTreeMap<id::Id, VideoTextures>,
format: wgpu::TextureFormat,
videos: BTreeMap<id::Id, VideoFrameData>,
}
pub trait HdrTextureFormatExt {
fn is_hdr(&self) -> bool;
pub trait WideTextureFormatExt {
fn is_wide(&self) -> bool;
}
impl HdrTextureFormatExt for wgpu::TextureFormat {
fn is_hdr(&self) -> bool {
impl WideTextureFormatExt for wgpu::TextureFormat {
fn is_wide(&self) -> bool {
matches!(
self,
wgpu::TextureFormat::Rgba16Float
| wgpu::TextureFormat::Rgba32Float
| wgpu::TextureFormat::Rgb10a2Unorm
| wgpu::TextureFormat::Rgb10a2Uint
| wgpu::TextureFormat::P010
)
}
}
@@ -225,15 +487,14 @@ impl Pipeline for VideoPipeline {
where
Self: Sized,
{
if format.is_hdr() {
if format.is_wide() {
tracing::info!("HDR texture format detected: {:?}", format);
}
let shader_passthrough =
device.create_shader_module(wgpu::include_wgsl!("shaders/passthrough.wgsl"));
let bind_group_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
label: Some("iced-video-texture-bind-group-layout"),
entries: &[
// y
wgpu::BindGroupLayoutEntry {
binding: 0,
visibility: wgpu::ShaderStages::FRAGMENT,
@@ -244,15 +505,40 @@ impl Pipeline for VideoPipeline {
},
count: None,
},
// uv
wgpu::BindGroupLayoutEntry {
binding: 1,
visibility: wgpu::ShaderStages::FRAGMENT,
ty: wgpu::BindingType::Texture {
multisampled: false,
view_dimension: wgpu::TextureViewDimension::D2,
sample_type: wgpu::TextureSampleType::Float { filterable: true },
},
count: None,
},
// sampler
wgpu::BindGroupLayoutEntry {
binding: 2,
visibility: wgpu::ShaderStages::FRAGMENT,
ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering),
count: None,
},
// conversion matrix
wgpu::BindGroupLayoutEntry {
binding: 3,
visibility: wgpu::ShaderStages::FRAGMENT,
ty: wgpu::BindingType::Buffer {
ty: wgpu::BufferBindingType::Uniform,
has_dynamic_offset: false,
min_binding_size: None,
},
count: None,
},
],
});
let shader_passthrough =
device.create_shader_module(wgpu::include_wgsl!("shaders/passthrough.wgsl"));
let render_pipeline_layout =
device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
label: Some("iced-video-render-pipeline-layout"),
@@ -273,7 +559,7 @@ impl Pipeline for VideoPipeline {
entry_point: Some("fs_main"),
targets: &[Some(wgpu::ColorTargetState {
format,
blend: Some(wgpu::BlendState::ALPHA_BLENDING),
blend: Some(wgpu::BlendState::REPLACE),
write_mask: wgpu::ColorWrites::ALL,
})],
compilation_options: wgpu::PipelineCompilationOptions::default(),

View File

@@ -1,31 +1,52 @@
// Vertex shader
// struct VertexOutput {
// @builtin(position) clip_position: vec4f,
// @location(0) coords: vec2f,
// }
// struct VertexInput {
// // @location(0) position: vec3<f32>,
// // @location(1) tex_coords: vec2<f32>,
// }
struct VertexOutput {
@builtin(position) clip_position: vec4<f32>,
@location(0) tex_coords: vec2<f32>,
};
}
@vertex
fn vs_main(
@builtin(vertex_index) in_vertex_index: u32,
) -> VertexOutput {
// model: VertexInput,
) -> VertexOutput {
var out: VertexOutput;
let uv = vec2<f32>(f32((in_vertex_index << 1u) & 2u), f32(in_vertex_index & 2u));
out.clip_position = vec4<f32>(uv * 2.0 - 1.0, 0.0, 1.0);
out.clip_position.y = -out.clip_position.y;
out.tex_coords = uv;
out.tex_coords = vec2<f32>(0.0, 0.0);
out.clip_position = vec4<f32>(0,0,0, 1.0);
return out;
}
// Fragment shader
@group(0) @binding(0)
var t_diffuse: texture_2d<f32>;
@group(0) @binding(1)
var s_diffuse: sampler;
// @vertex
// fn vs_main(@location(0) input: vec2f) -> VertexOutput {
// var out: VertexOutput;
// out.clip_position = vec4f(input, 0.0, 1.0);
// out.coords = input * 0.5 + vec2f(0.5, 0.5);
// return out;
// }
@group(0) @binding(0) var y_texture: texture_2d<f32>;
@group(0) @binding(1) var uv_texture: texture_2d<f32>;
@group(0) @binding(2) var texture_sampler: sampler;
@group(0) @binding(3) var<uniform> rgb_primaries: mat3x3<f32>;
@fragment
fn fs_main(in: VertexOutput) -> @location(0) vec4<f32> {
return textureSample(t_diffuse, s_diffuse, in.tex_coords);
fn fs_main(input: VertexOutput) -> @location(0) vec4<f32> {
let y = textureSample(y_texture, texture_sampler, input.tex_coords).r;
let uv = textureSample(uv_texture, texture_sampler, input.tex_coords).rg;
let yuv = vec3f(y, uv);
let rgb = rgb_primaries * yuv;
return vec4f(rgb, 1.0);
}

View File

@@ -206,6 +206,7 @@
apple-sdk_26
])
++ (lib.optionals pkgs.stdenv.isLinux [
ffmpeg
heaptrack
samply
cargo-flamegraph

View File

@@ -1,5 +1,7 @@
iced-video:
cd crates/iced-video && cargo run --release --example minimal
jello:
cargo r -r -- -vv
# iced-video:
# cd crates/iced-video && cargo run --release --example minimal
typegen:
@echo "Generating jellyfin type definitions..."
cd typegen && cargo run
@@ -10,6 +12,7 @@ hdrtest:
GST_DEBUG=3 gst-launch-1.0 playbin3 uri=https://jellyfin.tsuba.darksailor.dev/Items/6010382cf25273e624d305907010d773/Download?api_key=036c140222464878862231ef66a2bc9c video-sink="videoconvert ! video/x-raw,format=(string)RGB10A2_LE ! fakesink"
codec:
GST_DEBUG=3 gst-discoverer-1.0 -v https://jellyfin.tsuba.darksailor.dev/Items/6010382cf25273e624d305907010d773/Download?api_key=036c140222464878862231ef66a2bc9c
GST_DEBUG=3 gst-discoverer-1.0 https://jellyfin.tsuba.darksailor.dev/Items/6010382cf25273e624d305907010d773/Download?api_key=036c140222464878862231ef66a2bc9c
ffprobe:
ffprobe -v error -show_format -show_streams "https://jellyfin.tsuba.darksailor.dev/Items/6010382cf25273e624d305907010d773/Download?api_key=036c140222464878862231ef66a2bc9c" | grep pix_fmt

View File

@@ -1,6 +1,5 @@
use crate::*;
use iced::Element;
// mod widget;
pub fn settings(state: &State) -> Element<'_, Message> {
screens::settings(state)
@@ -20,6 +19,9 @@ pub fn update(state: &mut State, message: SettingsMessage) -> Task<Message> {
tracing::trace!("Switching settings screen to {:?}", screen);
state.settings.screen = screen;
}
SettingsMessage::User(user) => state.settings.login_form.update(user),
SettingsMessage::Server(server) => state.settings.server_form.update(server),
}
Task::none()
}
@@ -40,6 +42,28 @@ pub enum SettingsMessage {
Open,
Close,
Select(SettingsScreen),
User(UserMessage),
Server(ServerMessage),
}
#[derive(Debug, Clone)]
pub enum UserMessage {
Add,
UsernameChanged(String),
PasswordChanged(String),
// Edit(uuid::Uuid),
// Delete(uuid::Uuid),
Clear,
}
#[derive(Debug, Clone)]
pub enum ServerMessage {
Add,
NameChanged(String),
UrlChanged(String),
// Edit(uuid::Uuid),
// Delete(uuid::Uuid),
Clear,
}
#[derive(Debug, Clone, Default)]
@@ -66,14 +90,108 @@ pub struct UserItem {
#[derive(Debug, Clone, Default)]
pub struct LoginForm {
username: Option<String>,
password: Option<String>,
username: String,
password: String,
}
impl LoginForm {
pub fn update(&mut self, message: UserMessage) {
match message {
UserMessage::UsernameChanged(data) => {
self.username = data;
}
UserMessage::PasswordChanged(data) => {
self.password = data;
}
UserMessage::Add => {
// Handle adding user
}
UserMessage::Clear => {
self.username.clear();
self.password.clear();
}
}
}
pub fn view(&self) -> Element<'_, Message> {
iced::widget::column![
text("Login Form"),
text_input("Enter Username", &self.username).on_input(|data| {
Message::Settings(SettingsMessage::User(UserMessage::UsernameChanged(data)))
}),
text_input("Enter Password", &self.password)
.secure(true)
.on_input(|data| {
Message::Settings(SettingsMessage::User(UserMessage::PasswordChanged(data)))
}),
row![
button(text("Add User")).on_press_maybe(self.validate()),
button(text("Cancel"))
.on_press(Message::Settings(SettingsMessage::User(UserMessage::Clear))),
]
.spacing(10),
]
.spacing(10)
.padding([10, 0])
.into()
}
pub fn validate(&self) -> Option<Message> {
(!self.username.is_empty() && !self.password.is_empty())
.then(|| Message::Settings(SettingsMessage::User(UserMessage::Add)))
}
}
#[derive(Debug, Clone, Default)]
pub struct ServerForm {
name: Option<String>,
url: Option<String>,
name: String,
url: String,
}
impl ServerForm {
pub fn update(&mut self, message: ServerMessage) {
match message {
ServerMessage::NameChanged(data) => {
self.name = data;
}
ServerMessage::UrlChanged(data) => {
self.url = data;
}
ServerMessage::Add => {
// Handle adding server
}
ServerMessage::Clear => {
self.name.clear();
self.url.clear();
}
_ => {}
}
}
pub fn view(&self) -> Element<'_, Message> {
iced::widget::column![
text("Add New Server"),
text_input("Enter server name", &self.name).on_input(|data| {
Message::Settings(SettingsMessage::Server(ServerMessage::NameChanged(data)))
}),
text_input("Enter server URL", &self.url).on_input(|data| {
Message::Settings(SettingsMessage::Server(ServerMessage::UrlChanged(data)))
}),
row![
button(text("Add Server")).on_press_maybe(self.validate()),
button(text("Cancel")).on_press(Message::Settings(SettingsMessage::Server(
ServerMessage::Clear
))),
]
.spacing(10),
]
.spacing(10)
.padding([10, 0])
.into()
}
pub fn validate(&self) -> Option<Message> {
(!self.name.is_empty() && !self.url.is_empty())
.then(|| Message::Settings(SettingsMessage::Server(ServerMessage::Add)))
}
}
mod screens {
@@ -109,7 +227,6 @@ mod screens {
.map(|p| p.clip(true).width(Length::Fill).into()),
)
.width(Length::FillPortion(2))
// .max_width(Length::FillPortion(3))
.spacing(10)
.padding(10),
)
@@ -131,7 +248,8 @@ mod screens {
container(
Column::new()
.push(text("Server Settings"))
.push(toggler(false).label("Enable Server"))
.push(state.settings.server_form.view())
// .push(toggler(false).label("Enable Server"))
.spacing(20)
.padding(20),
)
@@ -141,7 +259,8 @@ mod screens {
container(
Column::new()
.push(text("User Settings"))
.push(toggler(true).label("Enable User"))
.push(state.settings.login_form.view())
// .push(userlist(&state))
.spacing(20)
.padding(20),
)