From 043d1e99f04504db165efbb431d8153c3177fb23 Mon Sep 17 00:00:00 2001 From: uttarayan21 Date: Mon, 22 Dec 2025 13:27:30 +0530 Subject: [PATCH] feat: Modify gst crate to add lot of more granularity --- Cargo.lock | 13 + Cargo.toml | 10 +- crates/iced-video/Cargo.toml | 12 + crates/iced-video/src/lib.rs | 123 +++++++ crates/iced-video/src/primitive.rs | 177 ++++++++++ crates/iced-video/src/shaders/bt.2020.wgsl | 0 crates/iced-video/src/shaders/bt.709.wgsl | 0 .../iced-video/src/shaders/passthrough.wgsl | 31 ++ crates/iced-video/src/source.rs | 64 ++++ gst/Cargo.toml | 3 +- gst/src/bin.rs | 9 +- gst/src/bus.rs | 26 ++ gst/src/caps.rs | 9 + gst/src/element.rs | 78 ++++- gst/src/isa.rs | 19 ++ gst/src/lib.rs | 316 ++---------------- gst/src/pad.rs | 10 + gst/src/pipeline.rs | 141 ++++++++ gst/src/plugins/app/appsink.rs | 127 +++++-- gst/src/plugins/playback/playbin3.rs | 154 +++++---- .../plugins/videoconvertscale/videoconvert.rs | 1 + gst/src/wgpu.rs | 5 +- ui-iced/Cargo.toml | 11 +- 23 files changed, 947 insertions(+), 392 deletions(-) create mode 100644 crates/iced-video/Cargo.toml create mode 100644 crates/iced-video/src/lib.rs create mode 100644 crates/iced-video/src/primitive.rs create mode 100644 crates/iced-video/src/shaders/bt.2020.wgsl create mode 100644 crates/iced-video/src/shaders/bt.709.wgsl create mode 100644 crates/iced-video/src/shaders/passthrough.wgsl create mode 100644 crates/iced-video/src/source.rs create mode 100644 gst/src/bus.rs create mode 100644 gst/src/isa.rs create mode 100644 gst/src/pipeline.rs diff --git a/Cargo.lock b/Cargo.lock index 0790a0d..74bb1a4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3050,6 +3050,7 @@ name = "gst" version = "0.1.0" dependencies = [ "error-stack", + "futures", "glib 0.21.5", "gstreamer 0.24.4", "gstreamer-app 0.24.4", @@ -3607,6 +3608,18 @@ dependencies = [ "thiserror 2.0.17", ] +[[package]] +name = "iced-video" +version = "0.1.0" +dependencies = [ + "error-stack", + "gst", + "iced_core", + "iced_wgpu", + "thiserror 2.0.17", + "tracing", +] + [[package]] name = "iced_beacon" version = "0.14.0" diff --git a/Cargo.toml b/Cargo.toml index 5c4ccb5..42f3337 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,16 +9,10 @@ members = [ "jello-types", "gst", "examples/hdr-gstreamer-wgpu", + "crates/iced-video", ] [workspace.dependencies] -iced = { version = "0.14.0", features = [ - "advanced", - "canvas", - "image", - "sipper", - "tokio", - "debug", -] } +iced = { version = "0.14.0" } iced_video_player = "0.6" gst = { version = "0.1.0", path = "gst" } # iced_video_player = { git = "https://github.com/jazzfool/iced_video_player" } diff --git a/crates/iced-video/Cargo.toml b/crates/iced-video/Cargo.toml new file mode 100644 index 0000000..1e2ba46 --- /dev/null +++ b/crates/iced-video/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "iced-video" +version = "0.1.0" +edition = "2024" + +[dependencies] +error-stack = "0.6.0" +gst.workspace = true +iced_core = "0.14.0" +iced_wgpu = "0.14.0" +thiserror = "2.0.17" +tracing = "0.1.43" diff --git a/crates/iced-video/src/lib.rs b/crates/iced-video/src/lib.rs new file mode 100644 index 0000000..5cab66b --- /dev/null +++ b/crates/iced-video/src/lib.rs @@ -0,0 +1,123 @@ +pub mod primitive; +pub mod source; + +use error_stack::{Report, ResultExt}; +use gst::*; +use iced_core::Length; +use std::marker::PhantomData; + +use gst::plugins::app::AppSink; +use gst::plugins::playback::Playbin3; +use gst::plugins::videoconvertscale::VideoConvert; + +#[derive(Debug, thiserror::Error)] +#[error("Iced Video Error")] +pub struct Error; +pub type Result> = core::result::Result; + +use std::sync::{Arc, Mutex, atomic::AtomicBool}; +pub struct Video { + id: iced_core::Id, + source: source::VideoSource, + is_playing: Arc, + is_eos: Arc, + texture: Mutex>, +} + +impl Video { + pub fn id(&self) -> &iced_core::Id { + &self.id + } + + pub fn source(&self) -> &source::VideoSource { + &self.source + } + + pub async fn new(url: impl AsRef) -> Result { + Ok(Self { + id: iced_core::Id::unique(), + source: source::VideoSource::new(url)?, + is_playing: Arc::new(AtomicBool::new(false)), + is_eos: Arc::new(AtomicBool::new(false)), + texture: Mutex::new(None), + }) + } +} + +pub struct VideoPlayer<'a, Message, Theme = iced_core::Theme, Renderer = iced_wgpu::Renderer> +where + Renderer: PrimitiveRenderer, +{ + videos: &'a Video, + content_fit: iced_core::ContentFit, + width: iced_core::Length, + height: iced_core::Length, + on_end_of_stream: Option, + on_new_frame: Option, + looping: bool, + // on_subtitle_text: Option) -> Message + 'a>>, + // on_error: Option Message + 'a>>, + theme: Theme, + __marker: PhantomData, +} + +impl VideoPlayer +where + Renderer: PrimitiveRenderer, +{ + pub fn new(source: source::VideoSource) -> Self { + Self { + videos: Video { + id: iced_core::Id::unique(), + source, + is_playing: Arc::new(AtomicBool::new(false)), + is_eos: Arc::new(AtomicBool::new(false)), + texture: Mutex::new(None), + }, + content_fit: iced_core::ContentFit::Contain, + width: Length::Shrink, + height: Length::Shrink, + on_end_of_stream: None, + on_new_frame: None, + looping: false, + theme: Theme::default(), + __marker: PhantomData, + } + } +} + +impl iced_core::Widget + for VideoPlayer<'_, Message, Theme, Renderer> +where + Message: Clone, + Renderer: PrimitiveRenderer, +{ + fn size(&self) -> iced_core::Size { + iced_core::Size { + width: self.width, + height: self.height, + } + } + + fn layout( + &mut self, + iced_core::widget::tree: &mut iced_core::widget::Tree, + iced_core::renderer: &Renderer, + limits: &iced_core::layout::Limits, + ) -> iced_core::layout::Node { + todo!() + } + + fn draw( + &self, + iced_core::widget::tree: &iced_core::widget::Tree, + iced_core::renderer: &mut Renderer, + theme: &Theme, + style: &iced_core::renderer::Style, + iced_core::layout: iced_core::Layout<'_>, + cursor: iced_core::mouse::Cursor, + viewport: &iced_core::Rectangle, + ) { + todo!() + } +} diff --git a/crates/iced-video/src/primitive.rs b/crates/iced-video/src/primitive.rs new file mode 100644 index 0000000..162304d --- /dev/null +++ b/crates/iced-video/src/primitive.rs @@ -0,0 +1,177 @@ +use iced_wgpu::primitive::Pipeline; +use iced_wgpu::wgpu; +use std::collections::BTreeMap; +use std::sync::{Arc, atomic::AtomicBool}; + +#[derive(Debug)] +pub struct VideoPrimitive { + texture: wgpu::TextureView, + ready: Arc, +} +impl iced_wgpu::Primitive for VideoPrimitive { + type Pipeline = VideoPipeline; + + fn prepare( + &self, + pipeline: &mut Self::Pipeline, + device: &wgpu::Device, + queue: &wgpu::Queue, + bounds: &iced_wgpu::core::Rectangle, + viewport: &iced_wgpu::graphics::Viewport, + ) { + todo!() + } + + fn draw(&self, _pipeline: &Self::Pipeline, _render_pass: &mut wgpu::RenderPass<'_>) -> bool { + false + } + + fn render( + &self, + pipeline: &Self::Pipeline, + encoder: &mut wgpu::CommandEncoder, + target: &wgpu::TextureView, + clip_bounds: &iced_wgpu::core::Rectangle, + ) { + if self.ready.load(std::sync::atomic::Ordering::SeqCst) { + let mut render_pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { + label: Some("iced-video-render-pass"), + color_attachments: &[Some(wgpu::RenderPassColorAttachment { + view: target, + resolve_target: None, + ops: wgpu::Operations { + load: wgpu::LoadOp::Load, + store: wgpu::StoreOp::Store, + }, + depth_slice: None, + })], + depth_stencil_attachment: None, + timestamp_writes: None, + occlusion_query_set: None, + }); + + render_pass.set_pipeline(&pipeline.pipeline); + render_pass.set_bind_group(0, &self.bind_group, &[]); + render_pass.draw(0..3, 0..1); + self.ready + .store(false, std::sync::atomic::Ordering::Relaxed); + } + } +} + +#[derive(Debug)] +pub struct VideoTextures { + id: u64, + texture: wgpu::Texture, + bind_group: wgpu::BindGroup, + ready: Arc, +} + +#[derive(Debug)] +pub struct VideoPipeline { + pipeline: wgpu::RenderPipeline, + bind_group_layout: wgpu::BindGroupLayout, + sampler: wgpu::Sampler, + videos: BTreeMap, +} + +pub trait HdrTextureFormatExt { + fn is_hdr(&self) -> bool; +} + +impl HdrTextureFormatExt for wgpu::TextureFormat { + fn is_hdr(&self) -> bool { + matches!( + self, + wgpu::TextureFormat::Rgba16Float + | wgpu::TextureFormat::Rgba32Float + | wgpu::TextureFormat::Rgb10a2Unorm + | wgpu::TextureFormat::Rgb10a2Uint + ) + } +} + +impl Pipeline for VideoPipeline { + fn new(device: &wgpu::Device, queue: &wgpu::Queue, format: wgpu::TextureFormat) -> Self + where + Self: Sized, + { + if format.is_hdr() { + tracing::info!("HDR texture format detected: {:?}", format); + } + let shader_passthrough = + device.create_shader_module(wgpu::include_wgsl!("shaders/passthrough.wgsl")); + + let bind_group_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("iced-video-texture-bind-group-layout"), + entries: &[ + wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + multisampled: false, + view_dimension: wgpu::TextureViewDimension::D2, + sample_type: wgpu::TextureSampleType::Float { filterable: true }, + }, + count: None, + }, + wgpu::BindGroupLayoutEntry { + binding: 1, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), + count: None, + }, + ], + }); + + let render_pipeline_layout = + device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("iced-video-render-pipeline-layout"), + bind_group_layouts: &[&bind_group_layout], + push_constant_ranges: &[], + }); + let pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("iced-video-render-pipeline"), + layout: Some(&render_pipeline_layout), + vertex: wgpu::VertexState { + module: &shader_passthrough, + entry_point: Some("vs_main"), + buffers: &[], + compilation_options: wgpu::PipelineCompilationOptions::default(), + }, + fragment: Some(wgpu::FragmentState { + module: &shader_passthrough, + entry_point: Some("fs_main"), + targets: &[Some(wgpu::ColorTargetState { + format, + blend: Some(wgpu::BlendState::ALPHA_BLENDING), + write_mask: wgpu::ColorWrites::ALL, + })], + compilation_options: wgpu::PipelineCompilationOptions::default(), + }), + primitive: wgpu::PrimitiveState::default(), + depth_stencil: None, + multisample: wgpu::MultisampleState::default(), + multiview: None, + cache: None, + }); + + let sampler = device.create_sampler(&wgpu::SamplerDescriptor { + label: Some("iced-video-sampler"), + address_mode_u: wgpu::AddressMode::ClampToEdge, + address_mode_v: wgpu::AddressMode::ClampToEdge, + address_mode_w: wgpu::AddressMode::ClampToEdge, + mag_filter: wgpu::FilterMode::Linear, + min_filter: wgpu::FilterMode::Linear, + mipmap_filter: wgpu::FilterMode::Nearest, + ..Default::default() + }); + + Self { + pipeline, + bind_group_layout, + sampler, + videos: BTreeMap::new(), + } + } +} diff --git a/crates/iced-video/src/shaders/bt.2020.wgsl b/crates/iced-video/src/shaders/bt.2020.wgsl new file mode 100644 index 0000000..e69de29 diff --git a/crates/iced-video/src/shaders/bt.709.wgsl b/crates/iced-video/src/shaders/bt.709.wgsl new file mode 100644 index 0000000..e69de29 diff --git a/crates/iced-video/src/shaders/passthrough.wgsl b/crates/iced-video/src/shaders/passthrough.wgsl new file mode 100644 index 0000000..1fd9706 --- /dev/null +++ b/crates/iced-video/src/shaders/passthrough.wgsl @@ -0,0 +1,31 @@ +// Vertex shader + +struct VertexOutput { + @builtin(position) clip_position: vec4, + @location(0) tex_coords: vec2, +}; + +@vertex +fn vs_main( + @builtin(vertex_index) in_vertex_index: u32, +) -> VertexOutput { + var out: VertexOutput; + let uv = vec2(f32((in_vertex_index << 1u) & 2u), f32(in_vertex_index & 2u)); + out.clip_position = vec4(uv * 2.0 - 1.0, 0.0, 1.0); + out.clip_position.y = -out.clip_position.y; + out.tex_coords = uv; + return out; +} + +// Fragment shader + +@group(0) @binding(0) +var t_diffuse: texture_2d; +@group(0) @binding(1) +var s_diffuse: sampler; + +@fragment +fn fs_main(in: VertexOutput) -> @location(0) vec4 { + return textureSample(t_diffuse, s_diffuse, in.tex_coords); +} + diff --git a/crates/iced-video/src/source.rs b/crates/iced-video/src/source.rs new file mode 100644 index 0000000..0e3efda --- /dev/null +++ b/crates/iced-video/src/source.rs @@ -0,0 +1,64 @@ +#[derive(Debug, Clone)] +pub struct VideoSource { + playbin: Playbin3, + videoconvert: VideoConvert, + appsink: AppSink, + bus: Bus, +} + +impl VideoSource { + /// Creates a new video source from the given URL. + /// Since this doesn't have to parse the pipeline manually, we aren't sanitizing the URL for + /// now. + pub async fn new(url: impl AsRef) -> Result { + Gst::new(); + let videoconvert = VideoConvert::new("iced-video-convert").change_context(Error)?; + let appsink = AppSink::new("iced-video-sink").change_context(Error)?; + let video_sink = videoconvert.link(&appsink).change_context(Error)?; + let playbin = gst::plugins::playback::Playbin3::new("iced-video") + .change_context(Error)? + .with_uri(url.as_ref()) + .with_video_sink(&video_sink); + let bus = playbin.bus().change_context(Error)?; + playbin.wait_ready()?; + // let bus_stream = bus.stream(); + // bus_stream.find(|message| { + // let view = message.view(); + // if let gst::MessageView::StateChanged(change) = view { + // change.current() == gst::State::Ready + // } else { + // false + // } + // }); + + Ok(Self { + playbin, + videoconvert, + appsink, + bus, + }) + } + + pub fn play(&self) -> Result<()> { + self.playbin + .play() + .change_context(Error) + .attach("Failed to play video") + } + + pub fn pause(&self) -> Result<()> { + self.playbin + .pause() + .change_context(Error) + .attach("Failed to pause video") + } + + pub fn bus(&self) -> &Bus {} + // pub fn copy_frame_to_texture(&self, texture: wgpu::TextureView) -> Result<()> { + // let frame = self + // .appsink + // .try_pull_sample(core::time::Duration::from_millis(1))? + // .ok_or(Error) + // .attach("No video frame available")?; + // } +} diff --git a/gst/Cargo.toml b/gst/Cargo.toml index c8b54e8..3c12c17 100644 --- a/gst/Cargo.toml +++ b/gst/Cargo.toml @@ -1,12 +1,13 @@ [package] name = "gst" version = "0.1.0" -edition = "2021" +edition = "2024" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] error-stack = "0.6" +futures = "0.3.31" glib = "0.21.5" gstreamer = { version = "0.24.4", features = ["v1_18"] } gstreamer-app = { version = "0.24.4", features = ["v1_18"] } diff --git a/gst/src/bin.rs b/gst/src/bin.rs index a12a641..94f6824 100644 --- a/gst/src/bin.rs +++ b/gst/src/bin.rs @@ -1,9 +1,16 @@ use crate::*; +#[repr(transparent)] pub struct Bin { inner: gstreamer::Bin, } +impl From for Bin { + fn from(inner: gstreamer::Bin) -> Self { + Bin { inner } + } +} + impl IsElement for Bin { fn as_element(&self) -> &Element { let element = self.inner.upcast_ref::(); @@ -22,7 +29,7 @@ impl Bin { Bin { inner: bin } } - pub fn add(&mut self, element: impl IsElement) -> Result<&mut Self> { + pub fn add(&mut self, element: &impl IsElement) -> Result<&mut Self> { self.inner .add(&element.as_element().inner) .change_context(Error) diff --git a/gst/src/bus.rs b/gst/src/bus.rs new file mode 100644 index 0000000..bc4bb61 --- /dev/null +++ b/gst/src/bus.rs @@ -0,0 +1,26 @@ +#[derive(Debug, Clone)] +#[repr(transparent)] +pub struct Bus { + pub(crate) bus: gstreamer::Bus, +} + +impl Bus { + pub fn iter_timed( + &self, + timeout: impl Into>, + ) -> gstreamer::bus::Iter<'_> { + let clocktime = match timeout.into() { + Some(dur) => gstreamer::ClockTime::try_from(dur).ok(), + None => gstreamer::ClockTime::NONE, + }; + self.bus.iter_timed(clocktime) + } + + pub fn stream(&self) -> gstreamer::bus::BusStream { + self.bus.stream() + } + + pub fn into_inner(self) -> gstreamer::Bus { + self.bus + } +} diff --git a/gst/src/caps.rs b/gst/src/caps.rs index b9a8493..6e95ba7 100644 --- a/gst/src/caps.rs +++ b/gst/src/caps.rs @@ -51,3 +51,12 @@ impl CapsBuilder { } } } + +impl Caps { + pub fn format(&self) -> Option<&str> { + use gstreamer::prelude::*; + self.inner + .structure(0) + .and_then(|s| s.get::<&str>("format").ok()) + } +} diff --git a/gst/src/element.rs b/gst/src/element.rs index 2a66330..230351d 100644 --- a/gst/src/element.rs +++ b/gst/src/element.rs @@ -1,4 +1,4 @@ -use crate::{Error, Pad, Result, ResultExt}; +use crate::{Bin, Error, Pad, Result, ResultExt}; #[repr(transparent)] pub struct Element { pub(crate) inner: gstreamer::Element, @@ -24,29 +24,85 @@ impl IsElement for Element { } pub trait Sink: IsElement { - fn sink_pad(&self) -> Pad { + fn sink(&self, name: impl AsRef) -> Pad { use gstreamer::prelude::*; self.as_element() - .pad("sink") + .pad(name.as_ref()) .map(From::from) .expect("Sink element has no sink pad") } } pub trait Source: IsElement { - fn source_pad(&self) -> Pad { + fn source(&self, name: impl AsRef) -> Pad { use gstreamer::prelude::*; self.as_element() - .pad("src") + .pad(name.as_ref()) .map(From::from) .expect("Source element has no src pad") } - fn link(&self, sink: &S) -> Result<()> { + fn link(&self, sink: &S) -> Result + where + Self: Sized, + { use gstreamer::prelude::*; - self.as_element() - .inner - .link(&sink.as_element().inner) - .change_context(Error) - .attach("Failed to link source to sink") + if let Ok(bin) = self.as_element().inner.clone().downcast::() { + bin.add(&sink.as_element().inner) + .change_context(Error) + .attach("Failed to add sink to bin")?; + self.as_element() + .inner + .link(&sink.as_element().inner) + .change_context(Error) + .attach("Failed to link elements")?; + Ok(Bin::from(bin)) + } else { + let bin = gstreamer::Bin::builder() + .name(format!( + "{}-link-{}", + self.as_element().inner.name(), + sink.as_element().inner.name() + )) + .build(); + bin.add(&self.as_element().inner) + .change_context(Error) + .attach("Failed to add source to bin")?; + bin.add(&sink.as_element().inner) + .change_context(Error) + .attach("Failed to add sink to bin")?; + self.as_element() + .inner + .link(&sink.as_element().inner) + .change_context(Error) + .attach("Failed to link elements")?; + if let Some(sink_pad) = self.as_element().pad("sink") { + let ghost_pad = Pad::ghost(&sink_pad)?; + bin.add_pad(&ghost_pad.inner) + .change_context(Error) + .attach("Failed to add src pad to bin")?; + ghost_pad.activate(true)?; + } + Ok(From::from(bin)) + } } + + // fn link_pad(&self, sink: &S, src_pad_name: &str, sink_pad_name: &str) -> Result<()> { + // use gstreamer::prelude::*; + // let src_pad = self + // .as_element() + // .pad(src_pad_name) + // .ok_or(Error) + // .attach("Source pad not found")?; + // let sink_pad = sink + // .as_element() + // .pad(sink_pad_name) + // .ok_or(Error) + // .attach("Sink pad not found")?; + // src_pad + // .inner + // .link(&sink_pad.inner) + // .change_context(Error) + // .attach("Failed to link source pad to sink pad")?; + // Ok(()) + // } } diff --git a/gst/src/isa.rs b/gst/src/isa.rs new file mode 100644 index 0000000..4392bb4 --- /dev/null +++ b/gst/src/isa.rs @@ -0,0 +1,19 @@ +// use crate::errors::*; +// /// This trait is used for implementing parent traits methods on children. +// pub trait Upcastable { +// #[track_caller] +// fn upcast(self) -> T; +// fn upcast_ref(&self) -> &T; +// } +// +// // impl Upcastable for crate::playback::Playbin3 {} +// impl core::ops::Deref for C +// where +// C: Upcastable

, +// { +// type Target = P; +// +// fn deref(&self) -> &Self::Target { +// todo!() +// } +// } diff --git a/gst/src/lib.rs b/gst/src/lib.rs index ce2e21e..72a92d0 100644 --- a/gst/src/lib.rs +++ b/gst/src/lib.rs @@ -1,19 +1,40 @@ pub mod bin; +pub mod bus; pub mod caps; pub mod element; pub mod errors; +pub mod isa; pub mod pad; +pub mod pipeline; pub mod plugins; -// pub mod playbin3; -// pub mod videoconvert; pub use bin::*; +pub use bus::*; pub use caps::*; pub use element::*; pub use pad::*; +pub use pipeline::*; pub use plugins::*; -// pub use playbin3::*; -// pub use videoconvert::*; + +pub(crate) mod priv_prelude { + pub use crate::errors::*; + pub use crate::*; + pub use gstreamer::prelude::*; + #[track_caller] + pub fn duration_to_clocktime( + timeout: impl Into>, + ) -> Result> { + match timeout.into() { + Some(dur) => { + let clocktime = gstreamer::ClockTime::try_from(dur) + .change_context(Error) + .attach("Failed to convert duration to ClockTime")?; + Ok(Some(clocktime)) + } + None => Ok(None), + } + } +} use errors::*; use gstreamer::prelude::*; @@ -36,282 +57,13 @@ impl Gst { Arc::clone(&GST) } - pub fn pipeline_from_str(&self, s: &str) -> Result { - let pipeline = gstreamer::parse::launch(s).change_context(Error)?; - let pipeline = pipeline.downcast::(); - let pipeline = match pipeline { - Err(_e) => return Err(Error).attach("Failed to downcast to Pipeline"), - Ok(p) => p, - }; - Ok(Pipeline { inner: pipeline }) - } -} - -pub struct Pipeline { - inner: gstreamer::Pipeline, -} - -impl core::fmt::Debug for Pipeline { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - f.debug_struct("Pipeline") - .field("pipeline", &self.inner) - // .field("state", &self.pipeline.state(gstreamer::ClockTime::NONE)) - .finish() - } -} - -impl Drop for Pipeline { - fn drop(&mut self) { - let _ = self.inner.set_state(gstreamer::State::Null); - } -} - -impl Pipeline { - pub fn bus(&self) -> Result { - let bus = self - .inner - .bus() - .ok_or(Error) - .attach("Failed to get bus from pipeline")?; - Ok(Bus { bus }) - } - - pub fn play(&self) -> Result<()> { - self.inner - .set_state(gstreamer::State::Playing) - .change_context(Error) - .attach("Failed to set pipeline to Playing state")?; - Ok(()) - } - - pub fn pause(&self) -> Result<()> { - self.inner - .set_state(gstreamer::State::Paused) - .change_context(Error) - .attach("Failed to set pipeline to Paused state")?; - Ok(()) - } - - pub fn ready(&self) -> Result<()> { - self.inner - .set_state(gstreamer::State::Ready) - .change_context(Error) - .attach("Failed to set pipeline to Paused state")?; - Ok(()) - } - - pub unsafe fn set_state( - &self, - state: gstreamer::State, - ) -> Result { - let result = self - .inner - .set_state(state) - .change_context(Error) - .attach("Failed to set pipeline state")?; - Ok(result) - } -} - -pub struct Bus { - bus: gstreamer::Bus, -} - -impl Bus { - pub fn iter_timed( - &self, - timeout: impl Into>, - ) -> gstreamer::bus::Iter<'_> { - let clocktime = match timeout.into() { - Some(dur) => gstreamer::ClockTime::try_from(dur).ok(), - None => gstreamer::ClockTime::NONE, - }; - self.bus.iter_timed(clocktime) - } - - pub fn stream(&self) -> gstreamer::bus::BusStream { - self.bus.stream() - } -} - -pub struct Playbin3Builder { - uri: Option, - video_sink: Option, - audio_sink: Option, - text_sink: Option, -} - -#[test] -fn gst_parse_pipeline() { - let gst = Gst::new(); - let pipeline = gst - .pipeline_from_str("videotestsrc ! autovideosink") - .expect("Failed to create pipeline"); - println!("{:?}", pipeline); -} - -#[test] -fn gst_parse_invalid_pipeline() { - let gst = Gst::new(); - let result = gst.pipeline_from_str("invalidpipeline"); - assert!(result.is_err()); -} - -#[test] -fn gst_play_pipeline() { - let gst = Gst::new(); - let pipeline = gst - .pipeline_from_str("videotestsrc ! autovideosink") - .expect("Failed to create pipeline"); - let bus = pipeline.bus().expect("Failed to get bus from pipeline"); - - pipeline - .play() - .expect("Unable to set the pipeline to the `Playing` state"); - - for msg in bus.iter_timed(None) { - use gstreamer::MessageView; - - match msg.view() { - MessageView::Eos(..) => break, - MessageView::Error(err) => { - eprintln!( - "Error from {:?}: {} ({:?})", - err.src().map(|s| s.path_string()), - err.error(), - err.debug() - ); - break; - } - _ => (), - } - } -} - -#[test] -#[ignore] -fn gstreamer_unwrapped() { - gstreamer::init(); - let uri = "https://gstreamer.freedesktop.org/data/media/sintel_trailer-480p.webm"; - let pipeline = gstreamer::parse::launch(&format!("playbin uri={}", uri)).unwrap(); - use gstreamer::prelude::*; - - pipeline.set_state(gstreamer::State::Playing).unwrap(); - - let bus = pipeline.bus().unwrap(); - for msg in bus.iter_timed(gstreamer::ClockTime::NONE) { - use gstreamer::MessageView; - - match msg.view() { - MessageView::Eos(..) => break, - MessageView::Error(err) => { - eprintln!( - "Error from {:?}: {} ({:?})", - err.src().map(|s| s.path_string()), - err.error(), - err.debug() - ); - break; - } - _ => (), - } - } - - pipeline.set_state(gstreamer::State::Null).unwrap(); -} - -#[test] -fn test_appsink() { - let gst = Gst::new(); - let pipeline = gst - .pipeline_from_str( - "videotestsrc ! videoconvert | capsfilter name=video-filter ! appsink name=video-sink", - ) - .expect("Failed to create pipeline"); - - // let video_sink = pipeline. - - let bus = pipeline.bus().expect("Failed to get bus from pipeline"); - - let sink = pipeline - .inner - .by_name("video-sink") - .expect("Sink not found") - .downcast::() - .expect("Failed to downcast to AppSink"); - let capsfilter = pipeline - .inner - .by_name("video-filter") - .expect("Capsfilter not found"); - - let caps = gstreamer::Caps::builder("video/x-raw") - .field("format", "RGBA") - .build(); - capsfilter.set_property("caps", &caps); - - sink.set_callbacks( - gstreamer_app::AppSinkCallbacks::builder() - .new_sample(|sink| { - // foo - Ok(gstreamer::FlowSuccess::Ok) - }) - .build(), - ); - - pipeline - .play() - .expect("Unable to set the pipeline to the `Playing` state"); - - for msg in bus.iter_timed(None) { - use gstreamer::MessageView; - - match msg.view() { - MessageView::Eos(..) => break, - MessageView::Error(err) => { - eprintln!( - "Error from {:?}: {} ({:?})", - err.src().map(|s| s.path_string()), - err.error(), - err.debug() - ); - break; - } - _ => (), - } - } -} - -#[test] -fn gst_test_manual_pipeline() { - use gstreamer as gst; - use gstreamer::prelude::*; - // Initialize GStreamer - gst::init().unwrap(); - - // Create a new pipeline - let pipeline = gst::Pipeline::new(); - - // Create elements for the pipeline - let src = gst::ElementFactory::make("videotestsrc").build().unwrap(); - let sink = gst::ElementFactory::make("autovideosink").build().unwrap(); - - // Add elements to the pipeline - pipeline.add_many(&[&src, &sink]).unwrap(); - - // Link elements together - src.link(&sink).unwrap(); - - // Set the pipeline to the playing state - pipeline.set_state(gst::State::Playing).unwrap(); - - // Start the main event loop - // let main_loop = glib::MainLoop::new(None, false); - // main_loop.run(); - // Shut down the pipeline and GStreamer - let bus = pipeline.bus().unwrap(); - let messages = bus.iter_timed(gst::ClockTime::NONE); - for msg in messages { - dbg!(msg); - } - pipeline.set_state(gst::State::Null).unwrap(); + // pub fn pipeline_from_str(&self, s: &str) -> Result { + // let pipeline = gstreamer::parse::launch(s).change_context(Error)?; + // let pipeline = pipeline.downcast::(); + // let pipeline = match pipeline { + // Err(_e) => return Err(Error).attach("Failed to downcast to Pipeline"), + // Ok(p) => p, + // }; + // Ok(Pipeline { inner: pipeline }) + // } } diff --git a/gst/src/pad.rs b/gst/src/pad.rs index 3081434..f947774 100644 --- a/gst/src/pad.rs +++ b/gst/src/pad.rs @@ -20,6 +20,16 @@ impl Pad { inner: ghost_pad.upcast(), }) } + + pub fn link(&self, peer: &Pad) -> Result<()> { + use gstreamer::prelude::*; + self.inner + .link(&peer.inner) + .change_context(Error) + .attach("Failed to link pads")?; + Ok(()) + } + pub fn activate(&self, activate: bool) -> Result<()> { use gstreamer::prelude::*; self.inner diff --git a/gst/src/pipeline.rs b/gst/src/pipeline.rs new file mode 100644 index 0000000..2a88acf --- /dev/null +++ b/gst/src/pipeline.rs @@ -0,0 +1,141 @@ +use crate::{playback::Playbin3, priv_prelude::*}; +use gstreamer::State; + +#[repr(transparent)] +pub struct Pipeline { + inner: gstreamer::Pipeline, +} + +impl core::fmt::Debug for Pipeline { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_struct("Pipeline") + .field("pipeline", &self.inner) + // .field("state", &self.pipeline.state(gstreamer::ClockTime::NONE)) + .finish() + } +} + +impl Drop for Pipeline { + fn drop(&mut self) { + let _ = self.inner.set_state(gstreamer::State::Null); + } +} + +impl Pipeline { + pub fn bus(&self) -> Result { + let bus = self + .inner + .bus() + .ok_or(Error) + .attach("Failed to get bus from pipeline")?; + Ok(Bus { bus }) + } + + /// Get the state + pub fn state( + &self, + timeout: impl Into>, + ) -> Result { + let (result, current, pending) = self.inner.state(duration_to_clocktime(timeout)?); + result.change_context(Error).attach("Failed to get state")?; + Ok(current) + } + + pub fn wait_non_null_sync(&self) -> Result<()> { + if self + .state(core::time::Duration::ZERO) + .change_context(Error) + .attach("Failed to get video context")? + != gstreamer::State::Null + { + Ok(()) + } else { + let bus = self.bus()?; + for message in bus.iter_timed(core::time::Duration::from_secs(1)) { + let view = message.view(); + dbg!(&view); + if let gstreamer::MessageView::StateChanged(change) = view + && change.current() != State::Null + { + break; + } + } + Ok(()) + } + } + + /// Waits for the pipeline to be ready + pub async fn wait_non_null(&self) -> Result<()> { + if self + .state(None) + .change_context(Error) + .attach("Failed to get video context")? + != gstreamer::State::Null + { + Ok(()) + } else { + use futures::StreamExt; + self.bus()? + .stream() + .filter(|message: &gstreamer::Message| { + let view = message.view(); + if let gstreamer::MessageView::StateChanged(change) = view { + core::future::ready(change.current() != gstreamer::State::Null) + } else { + core::future::ready(false) + } + }) + .next() + .await; + Ok(()) + } + } + + pub fn play(&self) -> Result<()> { + self.inner + .set_state(gstreamer::State::Playing) + .change_context(Error) + .attach("Failed to set pipeline to Playing state")?; + Ok(()) + } + + pub fn pause(&self) -> Result<()> { + self.inner + .set_state(gstreamer::State::Paused) + .change_context(Error) + .attach("Failed to set pipeline to Paused state")?; + Ok(()) + } + + pub fn ready(&self) -> Result<()> { + self.inner + .set_state(gstreamer::State::Ready) + .change_context(Error) + .attach("Failed to set pipeline to Paused state")?; + Ok(()) + } + + pub unsafe fn set_state( + &self, + state: gstreamer::State, + ) -> Result { + let result = self + .inner + .set_state(state) + .change_context(Error) + .attach("Failed to set pipeline state")?; + Ok(result) + } +} + +impl core::ops::Deref for Playbin3 { + type Target = Pipeline; + + fn deref(&self) -> &Self::Target { + let gp = self + .inner + .downcast_ref::() + .expect("BUG: Playbin3 must be a pipeline"); + unsafe { &*(gp as *const _ as *const Pipeline) } + } +} diff --git a/gst/src/plugins/app/appsink.rs b/gst/src/plugins/app/appsink.rs index 2cc5b94..38e32d1 100644 --- a/gst/src/plugins/app/appsink.rs +++ b/gst/src/plugins/app/appsink.rs @@ -1,5 +1,6 @@ -use crate::*; +use crate::priv_prelude::*; +#[derive(Debug, Clone)] pub struct AppSink { inner: gstreamer::Element, } @@ -32,9 +33,9 @@ impl AppSink { Ok(AppSink { inner }) } - pub fn with_caps(mut self, caps: &gstreamer::Caps) -> Self { + pub fn with_caps(mut self, caps: Caps) -> Self { use gstreamer::prelude::*; - // self.inner.set_caps(Some(caps)); + self.inner.set_property("caps", caps.inner); self } @@ -43,7 +44,7 @@ impl AppSink { Ok(()) } - pub fn pull_sample(&self, timeout: impl Into>) -> Result { + pub fn pull_sample(&self) -> Result { use gstreamer::prelude::*; self.appsink() .pull_sample() @@ -62,7 +63,7 @@ impl AppSink { .map(From::from)) } - pub fn pull_preroll(&self, timeout: impl Into>) -> Result { + pub fn pull_preroll(&self) -> Result { use gstreamer::prelude::*; self.appsink() .pull_preroll() @@ -83,26 +84,106 @@ impl AppSink { } } -fn duration_to_clocktime( - timeout: impl Into>, -) -> Result> { - match timeout.into() { - Some(dur) => { - let clocktime = gstreamer::ClockTime::try_from(dur) - .change_context(Error) - .attach("Failed to convert duration to ClockTime")?; - Ok(Some(clocktime)) - } - None => Ok(None), - } -} - -pub struct Sample { - inner: gstreamer::Sample, -} - impl From for Sample { fn from(inner: gstreamer::Sample) -> Self { Sample { inner } } } + +#[repr(transparent)] +pub struct Sample { + inner: gstreamer::Sample, +} + +use gstreamer::BufferRef; +impl Sample { + pub fn buffer(&self) -> Option<&BufferRef> { + self.inner.buffer() + } + + pub fn caps(&self) -> Option<&gstreamer::CapsRef> { + self.inner.caps() + } +} + +#[test] +fn test_appsink() { + use gstreamer::prelude::*; + use tracing_subscriber::prelude::*; + tracing_subscriber::registry() + .with( + tracing_subscriber::fmt::layer() + .with_thread_ids(true) + .with_file(true), + ) + .init(); + tracing::info!("Linking videoconvert to appsink"); + Gst::new(); + let playbin3 = playback::Playbin3::new("pppppppppppppppppppppppppppppp").unwrap().with_uri("https://jellyfin.tsuba.darksailor.dev/Items/6010382cf25273e624d305907010d773/Download?api_key=036c140222464878862231ef66a2bc9c"); + + let video_convert = plugins::videoconvertscale::VideoConvert::new("vcvcvcvcvcvcvcvcvcvcvcvcvc") + .expect("Create videoconvert"); + let appsink = app::AppSink::new("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") + .expect("Create appsink") + .with_caps( + Caps::builder(CapsType::Video) + .field("format", "RGB") + .build(), + ); + + let mut video_sink = video_convert + .link(&appsink) + .expect("Link videoconvert to appsink"); + + let playbin3 = playbin3.with_video_sink(&video_sink); + let bus = playbin3.bus().unwrap(); + // playbin3.play().expect("Play playbin3"); + + std::thread::spawn({ + let playbin3 = playbin3.clone(); + move || { + loop { + std::thread::sleep(std::time::Duration::from_secs(5)); + playbin3.play(); + playbin3.wait_non_null_sync(); + let sample = appsink + .try_pull_sample(core::time::Duration::from_secs(5)) + .expect("Pull sample from appsink") + .expect("No sample received from appsink"); + + dbg!(sample.caps()); + + playbin3.play(); + tracing::info!("Played"); + } + } + }); + for msg in bus.iter_timed(None) { + match msg.view() { + gstreamer::MessageView::Eos(..) => { + tracing::info!("End of stream reached"); + break; + } + gstreamer::MessageView::Error(err) => { + tracing::error!( + "Error from {:?}: {} ({:?})", + err.src().map(|s| s.path_string()), + err.error(), + err.debug() + ); + break; + } + gstreamer::MessageView::StateChanged(state) => { + eprintln!( + "State changed from {:?} to \x1b[33m{:?}\x1b[0m for {:?}", + state.old(), + state.current(), + state.src().map(|s| s.path_string()) + ); + } + _ => {} + } + // tracing::info!("{:#?}", &msg.view()); + } + // std::thread::sleep(std::time::Duration::from_secs(5)); +} diff --git a/gst/src/plugins/playback/playbin3.rs b/gst/src/plugins/playback/playbin3.rs index c0aef9c..f003e21 100644 --- a/gst/src/plugins/playback/playbin3.rs +++ b/gst/src/plugins/playback/playbin3.rs @@ -1,6 +1,9 @@ -use crate::*; +use crate::priv_prelude::*; + +#[derive(Debug, Clone)] +#[repr(transparent)] pub struct Playbin3 { - inner: gstreamer::Element, + pub(crate) inner: gstreamer::Element, } impl Drop for Playbin3 { @@ -61,65 +64,92 @@ impl Playbin3 { self.inner.property::("volume") } - pub fn play(&self) -> Result<()> { - use gstreamer::prelude::*; - self.inner - .set_state(gstreamer::State::Playing) - .change_context(Error) - .attach("Failed to set playbin3 to Playing state")?; - Ok(()) - } - pub fn bus(&self) -> Result { - let bus = self - .inner - .bus() - .ok_or(Error) - .attach("Failed to get bus from playbin3")?; - Ok(Bus { bus }) - } + // pub fn play(&self) -> Result<()> { + // use gstreamer::prelude::*; + // self.inner + // .set_state(gstreamer::State::Playing) + // .change_context(Error) + // .attach("Failed to set playbin3 to Playing state")?; + // Ok(()) + // } + // + // pub fn pause(&self) -> Result<()> { + // use gstreamer::prelude::*; + // self.inner + // .set_state(gstreamer::State::Paused) + // .change_context(Error) + // .attach("Failed to set playbin3 to Paused state")?; + // Ok(()) + // } + // + // pub fn bus(&self) -> Result { + // let bus = self + // .inner + // .bus() + // .ok_or(Error) + // .attach("Failed to get bus from playbin3")?; + // Ok(Bus { bus }) + // } } -#[test] -fn test_playbin3() { - use gstreamer::prelude::*; - use tracing_subscriber::prelude::*; - tracing_subscriber::registry() - .with( - tracing_subscriber::fmt::layer() - .with_thread_ids(true) - .with_file(true), - ) - .init(); - tracing::info!("Linking videoconvert to appsink"); - gstreamer::init().unwrap(); - let playbin3 = Playbin3::new("test_playbin3").unwrap().with_uri("https://jellyfin.tsuba.darksailor.dev/Items/6010382cf25273e624d305907010d773/Download?api_key=036c140222464878862231ef66a2bc9c"); - // let mut video_sink = Bin::new("wgpu_video_sink"); - // - // let video_convert = plugins::videoconvertscale::VideoConvert::new("wgpu_video_convert") - // .expect("Create videoconvert"); - // let appsink = AppSink::new("test_appsink").expect("Create appsink"); - let appsink = plugins::autodetect::AutoVideoSink::new("test_autodetect_video_sink") - .expect("Create autodetect video sink"); - // video_convert - // .link(&appsink) - // .expect("Link videoconvert to appsink"); - // - // let sink_pad = video_convert.sink_pad(); - // let sink_pad = Pad::ghost(&sink_pad).expect("Create ghost pad from videoconvert src"); - // video_sink - // .add(appsink) - // .expect("Add appsink to video sink") - // .add(video_convert) - // .expect("Add videoconvert to video sink") - // .add_pad(&sink_pad) - // .expect("Add ghost pad to video sink"); - // sink_pad.activate(true).expect("Activate ghost pad"); - - let playbin3 = playbin3.with_video_sink(&appsink); - playbin3.play().unwrap(); - let bus = playbin3.bus().unwrap(); - for msg in bus.iter_timed(None) { - tracing::info!("{:#?}", &msg.view()); - } - // std::thread::sleep(std::time::Duration::from_secs(5)); -} +// #[test] +// fn test_playbin3() { +// use gstreamer::prelude::*; +// use tracing_subscriber::prelude::*; +// tracing_subscriber::registry() +// .with( +// tracing_subscriber::fmt::layer() +// .with_thread_ids(true) +// .with_file(true), +// ) +// .init(); +// tracing::info!("Linking videoconvert to appsink"); +// Gst::new(); +// let playbin3 = Playbin3::new("pppppppppppppppppppppppppppppp").unwrap().with_uri("https://jellyfin.tsuba.darksailor.dev/Items/6010382cf25273e624d305907010d773/Download?api_key=036c140222464878862231ef66a2bc9c"); +// +// let video_convert = plugins::videoconvertscale::VideoConvert::new("vcvcvcvcvcvcvcvcvcvcvcvcvc") +// .expect("Create videoconvert"); +// let appsink = +// autodetect::AutoVideoSink::new("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").expect("Create appsink"); +// +// let mut video_sink = video_convert +// .link(&appsink) +// .expect("Link videoconvert to appsink"); +// +// let playbin3 = playbin3.with_video_sink(&video_sink); +// let bus = playbin3.bus().unwrap(); +// playbin3.play().expect("Play playbin3"); +// std::thread::spawn(move || loop { +// std::thread::sleep(std::time::Duration::from_secs(15)); +// playbin3.play(); +// tracing::info!("Played"); +// }); +// for msg in bus.iter_timed(None) { +// match msg.view() { +// gstreamer::MessageView::Eos(..) => { +// tracing::info!("End of stream reached"); +// break; +// } +// gstreamer::MessageView::Error(err) => { +// tracing::error!( +// "Error from {:?}: {} ({:?})", +// err.src().map(|s| s.path_string()), +// err.error(), +// err.debug() +// ); +// break; +// } +// gstreamer::MessageView::StateChanged(state) => { +// eprintln!( +// "State changed from {:?} to \x1b[33m{:?}\x1b[0m for {:?}", +// state.old(), +// state.current(), +// state.src().map(|s| s.path_string()) +// ); +// } +// _ => {} +// } +// // tracing::info!("{:#?}", &msg.view()); +// } +// // std::thread::sleep(std::time::Duration::from_secs(5)); +// } diff --git a/gst/src/plugins/videoconvertscale/videoconvert.rs b/gst/src/plugins/videoconvertscale/videoconvert.rs index c754c29..1bf578b 100644 --- a/gst/src/plugins/videoconvertscale/videoconvert.rs +++ b/gst/src/plugins/videoconvertscale/videoconvert.rs @@ -3,6 +3,7 @@ use crate::*; pub use gstreamer_video::VideoFormat; #[repr(transparent)] +#[derive(Debug, Clone)] pub struct VideoConvert { inner: gstreamer::Element, } diff --git a/gst/src/wgpu.rs b/gst/src/wgpu.rs index 1d645d6..139597f 100644 --- a/gst/src/wgpu.rs +++ b/gst/src/wgpu.rs @@ -1,3 +1,2 @@ -// pub fn copy_sample_to_texture() { -// -// } + + diff --git a/ui-iced/Cargo.toml b/ui-iced/Cargo.toml index b6060c4..d5501b6 100644 --- a/ui-iced/Cargo.toml +++ b/ui-iced/Cargo.toml @@ -9,7 +9,16 @@ api = { version = "0.1.0", path = "../api" } blurhash = "0.2.3" bytes = "1.11.0" gpui_util = "0.2.2" -iced = { workspace = true } +iced = { workspace = true, default-features = true, features = [ + "advanced", + "canvas", + "image", + "sipper", + "tokio", + "debug", +] } + + iced_video_player = { workspace = true } reqwest = "0.12.24" tap = "1.0.1"