From 4ed15c97f0e6e9d3556f5062ad3d8a572929a4b9 Mon Sep 17 00:00:00 2001 From: uttarayan21 Date: Thu, 25 Dec 2025 21:43:55 +0530 Subject: [PATCH] feat: Add keybinds to minimal example --- Cargo.lock | 2 +- crates/iced-video/examples/minimal.rs | 50 +++++++++++++++--- crates/iced-video/src/primitive.rs | 11 +--- crates/iced-video/src/source.rs | 69 +++++++++++++++++-------- examples/hdr-gstreamer-wgpu/Cargo.toml | 8 +-- examples/hdr-gstreamer-wgpu/src/main.rs | 28 +++++----- flake.nix | 3 +- gst/Cargo.toml | 6 +-- gst/src/lib.rs | 2 + gst/src/pipeline.rs | 15 ++++++ gst/src/plugins/app/appsink.rs | 67 +++++------------------- gst/src/plugins/playback.rs | 2 + gst/src/plugins/playback/playbin3.rs | 6 +++ 13 files changed, 156 insertions(+), 113 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7109e6f..81c7396 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3388,7 +3388,7 @@ dependencies = [ "gstreamer 0.24.4", "gstreamer-app 0.24.4", "gstreamer-base 0.24.4", - "gstreamer-video 0.23.6", + "gstreamer-video 0.24.4", "pollster 0.4.0", "tracing", "tracing-subscriber", diff --git a/crates/iced-video/examples/minimal.rs b/crates/iced-video/examples/minimal.rs index fab2a45..de891c1 100644 --- a/crates/iced-video/examples/minimal.rs +++ b/crates/iced-video/examples/minimal.rs @@ -10,7 +10,37 @@ pub fn main() -> iced::Result { ) .with(tracing_subscriber::EnvFilter::from_default_env()) .init(); - iced::application(State::new, update, view).run() + iced::application(State::new, update, view) + .subscription(keyboard_event) + .run() +} + +fn keyboard_event(state: &State) -> iced::Subscription { + use iced::keyboard::{Key, key::Named}; + iced::keyboard::listen().map(move |event| match event { + iced::keyboard::Event::KeyPressed { key, .. } => { + let key = key.as_ref(); + match key { + Key::Named(Named::Escape) | Key::Character("q") => Message::Quit, + Key::Named(Named::Space) => Message::Toggle, + _ => Message::Load, + } + // if key == &space { + // // Toggle play/pause + // let is_playing = state + // .video + // .source() + // .is_playing() + // .expect("Failed to get playing state"); + // if is_playing { + // Message::Pause + // } else { + // Message::Play + // } + // } + } + _ => Message::Load, + }) } #[derive(Debug, Clone)] @@ -22,6 +52,8 @@ impl State { pub fn new() -> Self { let video = VideoHandle::new("https://jellyfin.tsuba.darksailor.dev/Items/6010382cf25273e624d305907010d773/Download?api_key=036c140222464878862231ef66a2bc9c") .expect("Failed to create video handle"); + // let video = VideoHandle::new("file:///run/user/1000/gvfs/smb-share:server=tsuba.darksailor.dev,share=nas/Movies/Spider-Man - No Way Home (2021)/Spider-Man.No.Way.Home.2021.UHD.BluRay.2160p.TrueHD.Atmos.7.1.DV.HEVC.REMUX-FraMeSToR.mkv") + // .expect("Failed to create video handle"); Self { video } } } @@ -30,16 +62,18 @@ impl State { pub enum Message { Play, Pause, - Loaded, + Toggle, Load, + Quit, } pub fn update(state: &mut State, message: Message) -> iced::Task { match message { Message::Load => { // does stuff - let src = state.video.source().clone(); - iced::Task::perform(src.wait(), |_| Message::Loaded) + // let src = state.video.source().clone(); + // iced::Task::perform(src.wait(), |_| Message::Loaded) + iced::Task::none() } Message::Play => { state.video.source().play().expect("Failed to play video"); @@ -49,10 +83,14 @@ pub fn update(state: &mut State, message: Message) -> iced::Task { state.video.source().pause().expect("Failed to pause video"); iced::Task::none() } - Message::Loaded => { - // Video loaded + Message::Toggle => { + state.video.source().toggle().expect("Failed to stop video"); iced::Task::none() } + Message::Quit => { + state.video.source().stop().expect("Failed to stop video"); + std::process::exit(0); + } } } diff --git a/crates/iced-video/src/primitive.rs b/crates/iced-video/src/primitive.rs index bcf9e96..53ef70b 100644 --- a/crates/iced-video/src/primitive.rs +++ b/crates/iced-video/src/primitive.rs @@ -9,7 +9,7 @@ pub struct VideoFrame { pub id: id::Id, pub size: wgpu::Extent3d, pub ready: Arc, - pub frame: Arc>, + pub frame: Arc>, } impl iced_wgpu::Primitive for VideoFrame { @@ -97,7 +97,6 @@ impl iced_wgpu::Primitive for VideoFrame { video.bind_group = new_bind_group; } if video.ready.load(std::sync::atomic::Ordering::SeqCst) { - let now = std::time::Instant::now(); let frame = self.frame.lock().expect("BUG: Mutex poisoned"); let buffer = frame .buffer() @@ -110,7 +109,6 @@ impl iced_wgpu::Primitive for VideoFrame { video .ready .store(false, std::sync::atomic::Ordering::SeqCst); - tracing::info!("{:?} Taken to write to surface texture", now.elapsed()); } } @@ -148,12 +146,7 @@ impl iced_wgpu::Primitive for VideoFrame { view: target, resolve_target: None, ops: wgpu::Operations { - load: wgpu::LoadOp::Clear(wgpu::Color { - r: 0.1, - g: 0.2, - b: 0.3, - a: 1.0, - }), + load: wgpu::LoadOp::Load, store: wgpu::StoreOp::Store, }, depth_slice: None, diff --git a/crates/iced-video/src/source.rs b/crates/iced-video/src/source.rs index f61381b..5bb04f7 100644 --- a/crates/iced-video/src/source.rs +++ b/crates/iced-video/src/source.rs @@ -13,11 +13,11 @@ use std::sync::{Arc, Mutex, atomic::AtomicBool}; #[derive(Debug, Clone)] pub struct VideoSource { pub(crate) playbin: Playbin3, - // pub(crate) videoconvert: VideoConvert, + pub(crate) videoconvert: VideoConvert, pub(crate) appsink: AppSink, pub(crate) bus: Bus, pub(crate) ready: Arc, - pub(crate) frame: Arc>, + pub(crate) frame: Arc>, } impl VideoSource { @@ -26,31 +26,35 @@ impl VideoSource { /// now. pub fn new(url: impl AsRef) -> Result { Gst::new(); - // let videoconvert = VideoConvert::new("iced-video-convert") - // // .change_context(Error)? - // // .with_output_format(gst::plugins::videoconvertscale::VideoFormat::Rgba) - // .change_context(Error)?; - let appsink = AppSink::new("iced-video-sink") - .change_context(Error)? - .with_drop(true); - // .with_caps( - // Caps::builder(CapsType::Video) - // .field("format", "RGBA") - // .build(), - // ); - // let video_sink = videoconvert.link(&appsink).change_context(Error)?; + let videoconvert = VideoConvert::new("iced-video-convert") + // .change_context(Error)? + // .with_output_format(gst::plugins::videoconvertscale::VideoFormat::Rgba) + .change_context(Error)?; + let mut appsink = AppSink::new("iced-video-sink").change_context(Error)?; + appsink + .drop(true) + .sync(true) + .async_(true) + .emit_signals(true) + .caps( + Caps::builder(CapsType::Video) + .field("format", "RGBA") + .build(), + ); + let video_sink = videoconvert.link(&appsink).change_context(Error)?; let playbin = gst::plugins::playback::Playbin3::new("iced-video") .change_context(Error)? .with_uri(url.as_ref()) .with_buffer_duration(core::time::Duration::from_secs(2)) - .with_buffer_size(2000000) - .with_video_sink(&appsink); + .with_buffer_size(4096 * 4096 * 4 * 3) + .with_ring_buffer_max_size(4096 * 4096 * 4 * 3) + .with_video_sink(&video_sink); let bus = playbin.bus().change_context(Error)?; playbin.pause().change_context(Error)?; let ready = Arc::new(AtomicBool::new(false)); - let frame = Arc::new(Mutex::new(gst::app::Sample::new())); + let frame = Arc::new(Mutex::new(gst::Sample::new())); - let appsink = appsink.on_new_frame({ + appsink.on_new_frame({ let ready = Arc::clone(&ready); let frame = Arc::clone(&frame); move |appsink| { @@ -63,14 +67,13 @@ impl VideoSource { core::mem::replace(&mut *guard, sample); ready.store(true, std::sync::atomic::Ordering::Relaxed); } - Ok(()) } }); Ok(Self { playbin, - // videoconvert, + videoconvert, appsink, bus, ready, @@ -86,6 +89,23 @@ impl VideoSource { .attach("Failed to wait for video initialisation") } + pub fn is_playing(&self) -> Result { + let state = self + .playbin + .state(core::time::Duration::from_millis(0)) + .change_context(Error)?; + Ok(state == gst::State::Playing) + } + + pub fn toggle(&self) -> Result<()> { + if self.is_playing()? { + self.pause()?; + } else { + self.play()?; + } + Ok(()) + } + pub fn play(&self) -> Result<()> { self.playbin .play() @@ -100,6 +120,13 @@ impl VideoSource { .attach("Failed to pause video") } + pub fn stop(&self) -> Result<()> { + self.playbin + .stop() + .change_context(Error) + .attach("Failed to stop video") + } + pub fn size(&self) -> Result<(i32, i32)> { let caps = self .appsink diff --git a/examples/hdr-gstreamer-wgpu/Cargo.toml b/examples/hdr-gstreamer-wgpu/Cargo.toml index 62d0fb5..639204d 100644 --- a/examples/hdr-gstreamer-wgpu/Cargo.toml +++ b/examples/hdr-gstreamer-wgpu/Cargo.toml @@ -6,10 +6,10 @@ edition = "2024" [dependencies] # gst = { workspace = true } wgpu = "*" -gstreamer = "*" -gstreamer-video = "*" -gstreamer-app = "*" -gstreamer-base = "*" +gstreamer = { version = "0.24.4", features = ["v1_26"] } +gstreamer-app = { version = "0.24.4", features = ["v1_26"] } +gstreamer-base = { version = "0.24.4", features = ["v1_26"] } +gstreamer-video = { version = "0.24.4", features = ["v1_26"] } winit = { version = "*", features = ["wayland"] } anyhow = "*" pollster = "0.4.0" diff --git a/examples/hdr-gstreamer-wgpu/src/main.rs b/examples/hdr-gstreamer-wgpu/src/main.rs index fc6cc08..f504d21 100644 --- a/examples/hdr-gstreamer-wgpu/src/main.rs +++ b/examples/hdr-gstreamer-wgpu/src/main.rs @@ -77,7 +77,7 @@ impl State { .await .context("Failed to request wgpu device")?; let surface_caps = surface.get_capabilities(&adapter); - dbg!(&surface_caps); + tracing::info!("Caps: {:#?}", &surface_caps); let surface_format = surface_caps .formats .iter() @@ -85,6 +85,7 @@ impl State { .find(|f| f.is_hdr_format()) .expect("HDR format not supported") .clone(); + tracing::info!("Using surface format: {:?}", surface_format); let size = window.inner_size(); let config = wgpu::SurfaceConfiguration { usage: wgpu::TextureUsages::RENDER_ATTACHMENT, @@ -411,9 +412,8 @@ impl State { }, texture.size(), ); - drop(map); - // drop(buffer); - drop(frame); + // drop(map); + // drop(frame); Ok(()) } @@ -426,11 +426,11 @@ impl ApplicationHandler for App { let window = Arc::new(event_loop.create_window(window_attributes).unwrap()); - let monitor = event_loop - .primary_monitor() - .or_else(|| window.current_monitor()); + // let monitor = event_loop + // .primary_monitor() + // .or_else(|| window.current_monitor()); // window.set_fullscreen(None); - window.set_fullscreen(Some(winit::window::Fullscreen::Borderless(monitor))); + // window.set_fullscreen(Some(winit::window::Fullscreen::Borderless(monitor))); self.state = Some(pollster::block_on(State::new(window)).expect("Failed to block")); } @@ -528,7 +528,7 @@ impl Video { gst::init()?; use gst::prelude::*; let pipeline = gst::parse::launch( - r##"playbin3 uri=https://jellyfin.tsuba.darksailor.dev/Items/6010382cf25273e624d305907010d773/Download?api_key=036c140222464878862231ef66a2bc9c video-sink="videoconvert ! video/x-raw,format=RGB10A2_LE ! appsink name=appsink""##, + r##"playbin3 uri=https://jellyfin.tsuba.darksailor.dev/Items/6010382cf25273e624d305907010d773/Download?api_key=036c140222464878862231ef66a2bc9c video-sink="videoconvert ! video/x-raw,format=RGB10A2_LE ! appsink sync=true drop=true name=appsink""## ).context("Failed to parse gst pipeline")?; let pipeline = pipeline .downcast::() @@ -544,11 +544,11 @@ impl Video { })?; // appsink.set_property("max-buffers", 2u32); // appsink.set_property("emit-signals", true); - appsink.set_callbacks( - gst_app::AppSinkCallbacks::builder() - .new_sample(|_appsink| Ok(gst::FlowSuccess::Ok)) - .build(), - ); + // appsink.set_callbacks( + // gst_app::AppSinkCallbacks::builder() + // .new_sample(|_appsink| Ok(gst::FlowSuccess::Ok)) + // .build(), + // ); let bus = pipeline.bus().context("Failed to get gst pipeline bus")?; pipeline.set_state(gst::State::Playing)?; diff --git a/flake.nix b/flake.nix index f05d81a..a148d9b 100644 --- a/flake.nix +++ b/flake.nix @@ -202,8 +202,7 @@ apple-sdk_26 ]) ++ (lib.optionals pkgs.stdenv.isLinux [ - valgrind - hotspot + heaptrack samply cargo-flamegraph perf diff --git a/gst/Cargo.toml b/gst/Cargo.toml index 8d10b15..5cb1b0c 100644 --- a/gst/Cargo.toml +++ b/gst/Cargo.toml @@ -10,9 +10,9 @@ error-stack = "0.6" futures = "0.3.31" futures-lite = "2.6.1" glib = "0.21.5" -gstreamer = { version = "0.24.4", features = ["v1_18"] } -gstreamer-app = { version = "0.24.4", features = ["v1_18"] } -gstreamer-video = { version = "0.24.4", features = ["v1_18"] } +gstreamer = { version = "0.24.4", features = ["v1_26"] } +gstreamer-app = { version = "0.24.4", features = ["v1_26"] } +gstreamer-video = { version = "0.24.4", features = ["v1_26"] } thiserror = "2.0" tracing = { version = "0.1", features = ["log"] } diff --git a/gst/src/lib.rs b/gst/src/lib.rs index e4d4c79..fb3f313 100644 --- a/gst/src/lib.rs +++ b/gst/src/lib.rs @@ -8,6 +8,7 @@ pub mod pipeline; pub mod plugins; #[macro_use] pub mod wrapper; +pub mod sample; pub use bin::*; pub use bus::*; @@ -19,6 +20,7 @@ pub use gstreamer::{Message, MessageType, MessageView, State}; pub use pad::*; pub use pipeline::*; pub use plugins::*; +pub use sample::*; pub(crate) mod priv_prelude { pub use crate::errors::*; diff --git a/gst/src/pipeline.rs b/gst/src/pipeline.rs index 769f101..d7b08fc 100644 --- a/gst/src/pipeline.rs +++ b/gst/src/pipeline.rs @@ -60,6 +60,15 @@ impl Pipeline { Ok(()) } + #[track_caller] + pub fn stop(&self) -> Result<()> { + self.inner + .set_state(gstreamer::State::Null) + .change_context(Error) + .attach("Failed to set pipeline to Null state")?; + Ok(()) + } + #[track_caller] pub fn set_state(&self, state: gstreamer::State) -> Result { let result = self @@ -165,6 +174,12 @@ pub trait PipelineExt: ChildOf + Sync { fn ready(&self) -> Result<()> { self.upcast_ref().ready() } + + #[track_caller] + fn stop(&self) -> Result<()> { + self.upcast_ref().stop() + } + #[track_caller] fn set_state(&self, state: gstreamer::State) -> Result { self.upcast_ref().set_state(state) diff --git a/gst/src/plugins/app/appsink.rs b/gst/src/plugins/app/appsink.rs index d4a4dfc..3b333b7 100644 --- a/gst/src/plugins/app/appsink.rs +++ b/gst/src/plugins/app/appsink.rs @@ -25,41 +25,41 @@ impl AppSink { Ok(AppSink { inner }) } - pub fn with_emit_signals(self, emit: bool) -> Self { + pub fn emit_signals(&mut self, emit: bool) -> &mut Self { self.inner.set_property("emit-signals", emit); self } - pub fn with_async(self, async_: bool) -> Self { + pub fn async_(&mut self, async_: bool) -> &mut Self { self.inner.set_property("async", async_); self } - pub fn with_sync(self, sync: bool) -> Self { + pub fn sync(&mut self, sync: bool) -> &mut Self { self.inner.set_property("sync", sync); self } - pub fn with_drop(self, drop: bool) -> Self { + pub fn drop(&mut self, drop: bool) -> &mut Self { self.inner.set_property("drop", drop); self } - pub fn with_caps(self, caps: Caps) -> Self { + pub fn caps(&mut self, caps: Caps) -> &mut Self { self.inner.set_property("caps", caps.inner); self } - pub fn with_callbacks(self, callbacks: gstreamer_app::AppSinkCallbacks) -> Self { + pub fn callbacks(&mut self, callbacks: gstreamer_app::AppSinkCallbacks) -> &mut Self { self.appsink().set_callbacks(callbacks); self } - pub fn on_new_frame(self, mut f: F) -> Self + pub fn on_new_frame(&mut self, mut f: F) -> &mut Self where F: FnMut(&AppSink) -> Result<(), gstreamer::FlowError> + Send + 'static, { - self.with_emit_signals(true).with_callbacks( + self.emit_signals(true).callbacks( AppSinkCallbacks::builder() .new_sample(move |appsink| { use glib::object::Cast; @@ -109,44 +109,6 @@ impl AppSink { } } -impl From for Sample { - fn from(inner: gstreamer::Sample) -> Self { - Sample { inner } - } -} - -#[repr(transparent)] -#[derive(Debug, Clone)] -pub struct Sample { - pub inner: gstreamer::Sample, -} - -use gstreamer::BufferRef; -impl Sample { - #[doc(alias = "empty")] - pub fn new() -> Self { - Self { - inner: gstreamer::Sample::builder().build(), - } - } - - pub fn buffer(&self) -> Option<&BufferRef> { - self.inner.buffer() - } - - pub fn caps(&self) -> Option<&gstreamer::CapsRef> { - self.inner.caps() - } - - pub fn info(&self) -> Option<&gstreamer::StructureRef> { - self.inner.info() - } - - // pub fn set_buffer(&mut self) { - // self.inner.set_buffer(None); - // } -} - #[test] fn test_appsink() { use gstreamer::prelude::*; @@ -164,13 +126,12 @@ fn test_appsink() { let video_convert = plugins::videoconvertscale::VideoConvert::new("vcvcvcvcvcvcvcvcvcvcvcvcvc") .expect("Create videoconvert"); - let appsink = app::AppSink::new("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") - .expect("Create appsink") - .with_caps( - Caps::builder(CapsType::Video) - .field("format", "RGB") - .build(), - ); + let mut appsink = app::AppSink::new("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").expect("Create appsink"); + appsink.caps( + Caps::builder(CapsType::Video) + .field("format", "RGB") + .build(), + ); let mut video_sink = video_convert .link(&appsink) diff --git a/gst/src/plugins/playback.rs b/gst/src/plugins/playback.rs index c738c1c..3ba5734 100644 --- a/gst/src/plugins/playback.rs +++ b/gst/src/plugins/playback.rs @@ -1,2 +1,4 @@ pub mod playbin3; pub use playbin3::*; +pub mod playbin; +pub use playbin::*; diff --git a/gst/src/plugins/playback/playbin3.rs b/gst/src/plugins/playback/playbin3.rs index c0fd254..fa81e28 100644 --- a/gst/src/plugins/playback/playbin3.rs +++ b/gst/src/plugins/playback/playbin3.rs @@ -43,6 +43,12 @@ impl Playbin3 { self } + /// Sets the maximum size of the ring buffer in bytes. + pub fn with_ring_buffer_max_size(self, size: u64) -> Self { + self.inner.set_property("ring-buffer-max-size", size); + self + } + pub fn with_video_sink(self, video_sink: &impl ChildOf) -> Self { self.inner .set_property("video-sink", &video_sink.upcast_ref().inner);