diff --git a/Cargo.lock b/Cargo.lock index a7e316d..d0fa1bd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3050,6 +3050,7 @@ name = "gst" version = "0.1.0" dependencies = [ "error-stack", + "glib 0.21.5", "gstreamer 0.24.4", "gstreamer-app 0.24.4", "thiserror 2.0.17", diff --git a/examples/hdr-gstreamer-wgpu/src/main.rs b/examples/hdr-gstreamer-wgpu/src/main.rs index 79f153d..5d82fc7 100644 --- a/examples/hdr-gstreamer-wgpu/src/main.rs +++ b/examples/hdr-gstreamer-wgpu/src/main.rs @@ -43,7 +43,7 @@ pub struct State { window: Arc, gst: Video, surface: wgpu::Surface<'static>, - surface_texture: wgpu::Texture, + video_texture: wgpu::Texture, device: wgpu::Device, queue: wgpu::Queue, config: wgpu::SurfaceConfiguration, @@ -213,7 +213,7 @@ impl State { window, gst, surface, - surface_texture: video_texture, + video_texture, device, queue, config, @@ -239,13 +239,22 @@ impl State { return Ok(()); } + self.gst.poll(); + self.copy_next_frame_to_texture() .inspect_err(|e| { tracing::error!("Failed to copy video frame to texture: {e:?}"); }) .map_err(|_| wgpu::SurfaceError::Lost)?; - let output = self.surface.get_current_texture()?; + let output = match self.surface.get_current_texture() { + Ok(output) => output, + Err(wgpu::SurfaceError::Lost) => { + self.surface.configure(&self.device, &self.config); + return Ok(()); + } + Err(e) => return Err(e), + }; let view = output .texture .create_view(&wgpu::TextureViewDescriptor::default()); @@ -300,23 +309,90 @@ impl State { .context("Failed to get structure from caps")?; let width = size .get::("width") - .context("Failed to get width from caps")?; + .context("Failed to get width from caps")? as u32; let height = size .get::("height") - .context("Failed to get height from caps")?; + .context("Failed to get height from caps")? as u32; - if self.config.width != width as u32 || self.config.height != height as u32 { + let texture_size = self.video_texture.size(); + if texture_size.width != width || texture_size.height != height { tracing::info!( - "Resizing surface from {}x{} to {}x{}", - self.config.width, - self.config.height, + "Resizing video texture from {}x{} to {}x{}", + texture_size.width, + texture_size.height, width, height ); - self.resize(width as u32, height as u32); + self.video_texture = self.device.create_texture(&wgpu::TextureDescriptor { + size: wgpu::Extent3d { + width: width as u32, + height: height as u32, + depth_or_array_layers: 1, + }, + mip_level_count: 1, + sample_count: 1, + dimension: wgpu::TextureDimension::D2, + format: self.config.format, + usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, + label: Some("Jello Video Texture"), + view_formats: &[], + }); + let texture_bind_group_layout = + self.device + .create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("texture_bind_group_layout"), + entries: &[ + wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Texture { + multisampled: false, + view_dimension: wgpu::TextureViewDimension::D2, + sample_type: wgpu::TextureSampleType::Float { + filterable: true, + }, + }, + count: None, + }, + wgpu::BindGroupLayoutEntry { + binding: 1, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), + count: None, + }, + ], + }); + let sampler = self.device.create_sampler(&wgpu::SamplerDescriptor { + label: Some("texture_sampler"), + address_mode_u: wgpu::AddressMode::ClampToEdge, + address_mode_v: wgpu::AddressMode::ClampToEdge, + address_mode_w: wgpu::AddressMode::ClampToEdge, + mag_filter: wgpu::FilterMode::Linear, + min_filter: wgpu::FilterMode::Linear, + mipmap_filter: wgpu::FilterMode::Nearest, + ..Default::default() + }); + self.bind_group = self.device.create_bind_group(&wgpu::BindGroupDescriptor { + layout: &texture_bind_group_layout, + entries: &[ + wgpu::BindGroupEntry { + binding: 0, + resource: wgpu::BindingResource::TextureView( + &self + .video_texture + .create_view(&wgpu::TextureViewDescriptor::default()), + ), + }, + wgpu::BindGroupEntry { + binding: 1, + resource: wgpu::BindingResource::Sampler(&sampler), + }, + ], + label: Some("Jello Texture Bind Group"), + }); } + let texture = &self.video_texture; - let texture = &self.surface_texture; let buffer = frame.buffer().context("Failed to get buffer from sample")?; let map = buffer .map_readable() @@ -336,6 +412,9 @@ impl State { }, texture.size(), ); + drop(map); + // drop(buffer); + drop(frame); Ok(()) } @@ -348,21 +427,27 @@ impl ApplicationHandler for App { let window = Arc::new(event_loop.create_window(window_attributes).unwrap()); + let monitor = event_loop + .primary_monitor() + .or_else(|| window.current_monitor()); + // window.set_fullscreen(None); + window.set_fullscreen(Some(winit::window::Fullscreen::Borderless(monitor))); self.state = Some(pollster::block_on(State::new(window)).expect("Failed to block")); } - fn user_event(&mut self, _event_loop: &ActiveEventLoop, mut event: State) { + fn user_event(&mut self, _event_loop: &ActiveEventLoop, event: State) { self.state = Some(event); } fn about_to_wait(&mut self, _event_loop: &ActiveEventLoop) { let state = match &mut self.state { - Some(canvas) => canvas, + Some(state) => state, None => return, }; state.window.request_redraw(); } + fn window_event( &mut self, event_loop: &ActiveEventLoop, @@ -381,7 +466,6 @@ impl ApplicationHandler for App { state.resize(size.width, size.height) } WindowEvent::RedrawRequested => { - // dbg!("RedrawRequested"); // if state.gst.poll() { // event_loop.exit(); // return; @@ -459,8 +543,8 @@ impl Video { .map_err(|_| { anyhow::anyhow!("Failed to downcast video-sink appsink to gst_app::AppSink") })?; - // appsink.set_property("max-buffers", 10u32); - appsink.set_property("max-bytes", 10 * 4096 * 2160u64); + // appsink.set_property("max-buffers", 2u32); + // appsink.set_property("emit-signals", true); appsink.set_callbacks( gst_app::AppSinkCallbacks::builder() .new_sample(|_appsink| Ok(gst::FlowSuccess::Ok)) @@ -482,7 +566,7 @@ impl Video { pub fn poll(&mut self) -> bool { use gst::prelude::*; - for msg in self.bus.iter_timed(gst::ClockTime::NONE) { + for msg in self.bus.iter_timed(gst::ClockTime::ZERO) { use gst::MessageView; match msg.view() { diff --git a/flake.nix b/flake.nix index e03673c..97f8b39 100644 --- a/flake.nix +++ b/flake.nix @@ -195,6 +195,7 @@ cargo-outdated lld lldb + cargo-flamegraph ] ++ (lib.optionals pkgs.stdenv.isDarwin [ apple-sdk_26 diff --git a/gst/Cargo.toml b/gst/Cargo.toml index 5bff292..0fcda14 100644 --- a/gst/Cargo.toml +++ b/gst/Cargo.toml @@ -7,6 +7,7 @@ edition = "2021" [dependencies] error-stack = "0.6" +glib = "0.21.5" gstreamer = "0.24.4" gstreamer-app = "0.24.4" thiserror = "2.0" diff --git a/gst/src/lib.rs b/gst/src/lib.rs index e16cfb5..1e93a79 100644 --- a/gst/src/lib.rs +++ b/gst/src/lib.rs @@ -1,4 +1,5 @@ pub mod errors; +pub mod playbin3; use errors::*; use gstreamer::prelude::*; use std::sync::Arc; @@ -31,8 +32,11 @@ impl Gst { } } -pub struct Sink { - element: gstreamer::Element, +pub struct Bin { + bin: gstreamer::Bin, +} +pub struct Sample { + sample: gstreamer::Sample, } pub struct Pipeline { @@ -81,9 +85,13 @@ pub struct Bus { impl Bus { pub fn iter_timed( &self, - timeout: impl Into>, + timeout: impl Into>, ) -> gstreamer::bus::Iter<'_> { - self.bus.iter_timed(timeout) + let clocktime = match timeout.into() { + Some(dur) => gstreamer::ClockTime::try_from(dur).ok(), + None => gstreamer::ClockTime::NONE, + }; + self.bus.iter_timed(clocktime) } } @@ -96,6 +104,17 @@ pub struct Element { element: gstreamer::Element, } +pub struct AppSink { + appsink: gstreamer_app::AppSink, +} + +pub struct Playbin3Builder { + uri: Option, + video_sink: Option, + audio_sink: Option, + text_sink: Option, +} + #[test] fn gst_parse_pipeline() { let gst = Gst::new(); @@ -124,7 +143,7 @@ fn gst_play_pipeline() { .set_state(gstreamer::State::Playing) .expect("Unable to set the pipeline to the `Playing` state"); - for msg in bus.iter_timed(gstreamer::ClockTime::NONE) { + for msg in bus.iter_timed(None) { use gstreamer::MessageView; match msg.view() { @@ -225,7 +244,7 @@ fn test_appsink() { .set_state(gstreamer::State::Playing) .expect("Unable to set the pipeline to the `Playing` state"); - for msg in bus.iter_timed(gstreamer::ClockTime::NONE) { + for msg in bus.iter_timed(None) { use gstreamer::MessageView; match msg.view() { @@ -243,3 +262,38 @@ fn test_appsink() { } } } + +#[test] +fn gst_test_manual_pipeline() { + use gstreamer as gst; + use gstreamer::prelude::*; + // Initialize GStreamer + gst::init().unwrap(); + + // Create a new pipeline + let pipeline = gst::Pipeline::new(); + + // Create elements for the pipeline + let src = gst::ElementFactory::make("videotestsrc").build().unwrap(); + let sink = gst::ElementFactory::make("autovideosink").build().unwrap(); + + // Add elements to the pipeline + pipeline.add_many(&[&src, &sink]).unwrap(); + + // Link elements together + src.link(&sink).unwrap(); + + // Set the pipeline to the playing state + pipeline.set_state(gst::State::Playing).unwrap(); + + // Start the main event loop + // let main_loop = glib::MainLoop::new(None, false); + // main_loop.run(); + // Shut down the pipeline and GStreamer + let bus = pipeline.bus().unwrap(); + let messages = bus.iter_timed(gst::ClockTime::NONE); + for msg in messages { + dbg!(msg); + } + pipeline.set_state(gst::State::Null).unwrap(); +}