feat(gst): implement Playbin3 wrapper with basic playback functionality

Cargo.lock

@@ -3050,6 +3050,7 @@ name = "gst"
 version = "0.1.0"
 dependencies = [
  "error-stack",
+ "glib 0.21.5",
  "gstreamer 0.24.4",
  "gstreamer-app 0.24.4",
  "thiserror 2.0.17",

examples/hdr-gstreamer-wgpu/src/main.rs

@@ -43,7 +43,7 @@ pub struct State {
     window: Arc<Window>,
     gst: Video,
     surface: wgpu::Surface<'static>,
-    surface_texture: wgpu::Texture,
+    video_texture: wgpu::Texture,
     device: wgpu::Device,
     queue: wgpu::Queue,
     config: wgpu::SurfaceConfiguration,
@@ -213,7 +213,7 @@ impl State {
             window,
             gst,
             surface,
-            surface_texture: video_texture,
+            video_texture,
             device,
             queue,
             config,
@@ -239,13 +239,22 @@ impl State {
             return Ok(());
         }
 
+        self.gst.poll();
+
         self.copy_next_frame_to_texture()
             .inspect_err(|e| {
                 tracing::error!("Failed to copy video frame to texture: {e:?}");
             })
             .map_err(|_| wgpu::SurfaceError::Lost)?;
 
-        let output = self.surface.get_current_texture()?;
+        let output = match self.surface.get_current_texture() {
+            Ok(output) => output,
+            Err(wgpu::SurfaceError::Lost) => {
+                self.surface.configure(&self.device, &self.config);
+                return Ok(());
+            }
+            Err(e) => return Err(e),
+        };
         let view = output
             .texture
             .create_view(&wgpu::TextureViewDescriptor::default());
@@ -300,23 +309,90 @@ impl State {
             .context("Failed to get structure from caps")?;
         let width = size
             .get::<i32>("width")
-            .context("Failed to get width from caps")?;
+            .context("Failed to get width from caps")? as u32;
         let height = size
             .get::<i32>("height")
-            .context("Failed to get height from caps")?;
+            .context("Failed to get height from caps")? as u32;
 
-        if self.config.width != width as u32 || self.config.height != height as u32 {
+        let texture_size = self.video_texture.size();
+        if texture_size.width != width || texture_size.height != height {
             tracing::info!(
-                "Resizing surface from {}x{} to {}x{}",
+                "Resizing video texture from {}x{} to {}x{}",
-                self.config.width,
+                texture_size.width,
-                self.config.height,
+                texture_size.height,
                 width,
                 height
             );
-            self.resize(width as u32, height as u32);
+            self.video_texture = self.device.create_texture(&wgpu::TextureDescriptor {
+                size: wgpu::Extent3d {
+                    width: width as u32,
+                    height: height as u32,
+                    depth_or_array_layers: 1,
+                },
+                mip_level_count: 1,
+                sample_count: 1,
+                dimension: wgpu::TextureDimension::D2,
+                format: self.config.format,
+                usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
+                label: Some("Jello Video Texture"),
+                view_formats: &[],
+            });
+            let texture_bind_group_layout =
+                self.device
+                    .create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
+                        label: Some("texture_bind_group_layout"),
+                        entries: &[
+                            wgpu::BindGroupLayoutEntry {
+                                binding: 0,
+                                visibility: wgpu::ShaderStages::FRAGMENT,
+                                ty: wgpu::BindingType::Texture {
+                                    multisampled: false,
+                                    view_dimension: wgpu::TextureViewDimension::D2,
+                                    sample_type: wgpu::TextureSampleType::Float {
+                                        filterable: true,
+                                    },
+                                },
+                                count: None,
+                            },
+                            wgpu::BindGroupLayoutEntry {
+                                binding: 1,
+                                visibility: wgpu::ShaderStages::FRAGMENT,
+                                ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering),
+                                count: None,
+                            },
+                        ],
+                    });
+            let sampler = self.device.create_sampler(&wgpu::SamplerDescriptor {
+                label: Some("texture_sampler"),
+                address_mode_u: wgpu::AddressMode::ClampToEdge,
+                address_mode_v: wgpu::AddressMode::ClampToEdge,
+                address_mode_w: wgpu::AddressMode::ClampToEdge,
+                mag_filter: wgpu::FilterMode::Linear,
+                min_filter: wgpu::FilterMode::Linear,
+                mipmap_filter: wgpu::FilterMode::Nearest,
+                ..Default::default()
+            });
+            self.bind_group = self.device.create_bind_group(&wgpu::BindGroupDescriptor {
+                layout: &texture_bind_group_layout,
+                entries: &[
+                    wgpu::BindGroupEntry {
+                        binding: 0,
+                        resource: wgpu::BindingResource::TextureView(
+                            &self
+                                .video_texture
+                                .create_view(&wgpu::TextureViewDescriptor::default()),
+                        ),
+                    },
+                    wgpu::BindGroupEntry {
+                        binding: 1,
+                        resource: wgpu::BindingResource::Sampler(&sampler),
+                    },
+                ],
+                label: Some("Jello Texture Bind Group"),
+            });
         }
+        let texture = &self.video_texture;
 
-        let texture = &self.surface_texture;
         let buffer = frame.buffer().context("Failed to get buffer from sample")?;
         let map = buffer
             .map_readable()
@@ -336,6 +412,9 @@ impl State {
             },
             texture.size(),
         );
+        drop(map);
+        // drop(buffer);
+        drop(frame);
 
         Ok(())
     }
@@ -348,21 +427,27 @@ impl ApplicationHandler<State> for App {
 
         let window = Arc::new(event_loop.create_window(window_attributes).unwrap());
 
+        let monitor = event_loop
+            .primary_monitor()
+            .or_else(|| window.current_monitor());
+        // window.set_fullscreen(None);
+        window.set_fullscreen(Some(winit::window::Fullscreen::Borderless(monitor)));
         self.state = Some(pollster::block_on(State::new(window)).expect("Failed to block"));
     }
 
-    fn user_event(&mut self, _event_loop: &ActiveEventLoop, mut event: State) {
+    fn user_event(&mut self, _event_loop: &ActiveEventLoop, event: State) {
         self.state = Some(event);
     }
 
     fn about_to_wait(&mut self, _event_loop: &ActiveEventLoop) {
         let state = match &mut self.state {
-            Some(canvas) => canvas,
+            Some(state) => state,
             None => return,
         };
 
         state.window.request_redraw();
     }
+
     fn window_event(
         &mut self,
         event_loop: &ActiveEventLoop,
@@ -381,7 +466,6 @@ impl ApplicationHandler<State> for App {
                 state.resize(size.width, size.height)
             }
             WindowEvent::RedrawRequested => {
-                // dbg!("RedrawRequested");
                 // if state.gst.poll() {
                 //     event_loop.exit();
                 //     return;
@@ -459,8 +543,8 @@ impl Video {
             .map_err(|_| {
                 anyhow::anyhow!("Failed to downcast video-sink appsink to gst_app::AppSink")
             })?;
-        // appsink.set_property("max-buffers", 10u32);
+        // appsink.set_property("max-buffers", 2u32);
-        appsink.set_property("max-bytes", 10 * 4096 * 2160u64);
+        // appsink.set_property("emit-signals", true);
         appsink.set_callbacks(
             gst_app::AppSinkCallbacks::builder()
                 .new_sample(|_appsink| Ok(gst::FlowSuccess::Ok))
@@ -482,7 +566,7 @@ impl Video {
 
     pub fn poll(&mut self) -> bool {
         use gst::prelude::*;
-        for msg in self.bus.iter_timed(gst::ClockTime::NONE) {
+        for msg in self.bus.iter_timed(gst::ClockTime::ZERO) {
             use gst::MessageView;
 
             match msg.view() {

flake.nix

@@ -195,6 +195,7 @@
                     cargo-outdated
                     lld
                     lldb
+                    cargo-flamegraph
                   ]
                   ++ (lib.optionals pkgs.stdenv.isDarwin [
                     apple-sdk_26

gst/Cargo.toml

@@ -7,6 +7,7 @@ edition = "2021"
 
 [dependencies]
 error-stack = "0.6"
+glib = "0.21.5"
 gstreamer = "0.24.4"
 gstreamer-app = "0.24.4"
 thiserror = "2.0"

gst/src/lib.rs

@@ -1,4 +1,5 @@
 pub mod errors;
+pub mod playbin3;
 use errors::*;
 use gstreamer::prelude::*;
 use std::sync::Arc;
@@ -31,8 +32,11 @@ impl Gst {
     }
 }
 
-pub struct Sink {
+pub struct Bin {
-    element: gstreamer::Element,
+    bin: gstreamer::Bin,
+}
+pub struct Sample {
+    sample: gstreamer::Sample,
 }
 
 pub struct Pipeline {
@@ -81,9 +85,13 @@ pub struct Bus {
 impl Bus {
     pub fn iter_timed(
         &self,
-        timeout: impl Into<Option<gstreamer::ClockTime>>,
+        timeout: impl Into<Option<core::time::Duration>>,
     ) -> gstreamer::bus::Iter<'_> {
-        self.bus.iter_timed(timeout)
+        let clocktime = match timeout.into() {
+            Some(dur) => gstreamer::ClockTime::try_from(dur).ok(),
+            None => gstreamer::ClockTime::NONE,
+        };
+        self.bus.iter_timed(clocktime)
     }
 }
 
@@ -96,6 +104,17 @@ pub struct Element {
     element: gstreamer::Element,
 }
 
+pub struct AppSink {
+    appsink: gstreamer_app::AppSink,
+}
+
+pub struct Playbin3Builder {
+    uri: Option<String>,
+    video_sink: Option<Element>,
+    audio_sink: Option<Element>,
+    text_sink: Option<Element>,
+}
+
 #[test]
 fn gst_parse_pipeline() {
     let gst = Gst::new();
@@ -124,7 +143,7 @@ fn gst_play_pipeline() {
         .set_state(gstreamer::State::Playing)
         .expect("Unable to set the pipeline to the `Playing` state");
 
-    for msg in bus.iter_timed(gstreamer::ClockTime::NONE) {
+    for msg in bus.iter_timed(None) {
         use gstreamer::MessageView;
 
         match msg.view() {
@@ -225,7 +244,7 @@ fn test_appsink() {
         .set_state(gstreamer::State::Playing)
         .expect("Unable to set the pipeline to the `Playing` state");
 
-    for msg in bus.iter_timed(gstreamer::ClockTime::NONE) {
+    for msg in bus.iter_timed(None) {
         use gstreamer::MessageView;
 
         match msg.view() {
@@ -243,3 +262,38 @@ fn test_appsink() {
         }
     }
 }
+
+#[test]
+fn gst_test_manual_pipeline() {
+    use gstreamer as gst;
+    use gstreamer::prelude::*;
+    // Initialize GStreamer
+    gst::init().unwrap();
+
+    // Create a new pipeline
+    let pipeline = gst::Pipeline::new();
+
+    // Create elements for the pipeline
+    let src = gst::ElementFactory::make("videotestsrc").build().unwrap();
+    let sink = gst::ElementFactory::make("autovideosink").build().unwrap();
+
+    // Add elements to the pipeline
+    pipeline.add_many(&[&src, &sink]).unwrap();
+
+    // Link elements together
+    src.link(&sink).unwrap();
+
+    // Set the pipeline to the playing state
+    pipeline.set_state(gst::State::Playing).unwrap();
+
+    // Start the main event loop
+    // let main_loop = glib::MainLoop::new(None, false);
+    // main_loop.run();
+    // Shut down the pipeline and GStreamer
+    let bus = pipeline.bus().unwrap();
+    let messages = bus.iter_timed(gst::ClockTime::NONE);
+    for msg in messages {
+        dbg!(msg);
+    }
+    pipeline.set_state(gst::State::Null).unwrap();
+}

gst/src/playbin3.rs

@@ -0,0 +1,65 @@
+use crate::*;
+pub struct Playbin3 {
+    inner: gstreamer::Element,
+}
+
+impl Drop for Playbin3 {
+    fn drop(&mut self) {
+        let _ = self.inner.set_state(gstreamer::State::Null);
+    }
+}
+
+impl Playbin3 {
+    pub fn new(name: impl AsRef<str>) -> Result<Self> {
+        use gstreamer::prelude::*;
+        gstreamer::ElementFactory::make("playbin3")
+            .name(name.as_ref())
+            .build()
+            .map(|element| Playbin3 { inner: element })
+            .change_context(Error)
+    }
+
+    pub fn with_uri(self, uri: impl AsRef<str>) -> Self {
+        use gstreamer::prelude::*;
+        self.inner.set_property("uri", uri.as_ref());
+        self
+    }
+
+    pub fn sample(&self) -> Option<Sample> {
+        use gstreamer::prelude::*;
+        self.inner
+            .property::<Option<gstreamer::Sample>>("sample")
+            // // .and_then(|v| v.get::<gstreamer::Sample>().ok())
+            .map(|sample| Sample { sample })
+    }
+
+    pub fn play(&self) -> Result<()> {
+        use gstreamer::prelude::*;
+        self.inner
+            .set_state(gstreamer::State::Playing)
+            .change_context(Error)
+            .attach("Failed to set playbin3 to Playing state")?;
+        Ok(())
+    }
+    pub fn bus(&self) -> Result<Bus> {
+        let bus = self
+            .inner
+            .bus()
+            .ok_or(Error)
+            .attach("Failed to get bus from playbin3")?;
+        Ok(Bus { bus })
+    }
+}
+
+#[test]
+fn test_playbin3() {
+    use gstreamer::prelude::*;
+    gstreamer::init().unwrap();
+    let playbin3 = Playbin3::new("test_playbin3").unwrap().with_uri(
+        "https://www.freedesktop.org/software/gstreamer-sdk/data/media/sintel_trailer-480p.webm",
+    );
+    playbin3.play().unwrap();
+    let bus = playbin3.bus().unwrap();
+    for msg in bus.iter_timed(None) {}
+    // std::thread::sleep(std::time::Duration::from_secs(5));
+}