feat: Get iced-video working

crates/iced-video/src/source.rs

@@ -1,44 +1,107 @@
+use crate::{Error, Result, ResultExt};
+use gst::{
+    Bus, Gst, MessageType, MessageView, Sink, Source,
+    app::AppSink,
+    caps::{Caps, CapsType},
+    element::ElementExt,
+    pipeline::PipelineExt,
+    playback::Playbin3,
+    videoconvertscale::VideoConvert,
+};
+use std::sync::{Arc, Mutex, atomic::AtomicBool};
+
 #[derive(Debug, Clone)]
 pub struct VideoSource {
-    playbin: Playbin3,
-    videoconvert: VideoConvert,
-    appsink: AppSink,
-    bus: Bus,
+    pub(crate) playbin: Playbin3,
+    pub(crate) videoconvert: VideoConvert,
+    pub(crate) appsink: AppSink,
+    pub(crate) bus: Bus,
+    pub(crate) ready: Arc<AtomicBool>,
+    pub(crate) frame: Arc<Mutex<Vec<u8>>>,
 }
 
 impl VideoSource {
     /// Creates a new video source from the given URL.
     /// Since this doesn't have to parse the pipeline manually, we aren't sanitizing the URL for
     /// now.
-    pub async fn new(url: impl AsRef<str>) -> Result<Self> {
+    pub fn new(url: impl AsRef<str>) -> Result<Self> {
         Gst::new();
-        let videoconvert = VideoConvert::new("iced-video-convert").change_context(Error)?;
-        let appsink = AppSink::new("iced-video-sink").change_context(Error)?;
+        let videoconvert = VideoConvert::new("iced-video-convert")
+            // .change_context(Error)?
+            // .with_output_format(gst::plugins::videoconvertscale::VideoFormat::Rgba)
+            .change_context(Error)?;
+        let appsink = AppSink::new("iced-video-sink")
+            .change_context(Error)?
+            .with_caps(
+                Caps::builder(CapsType::Video)
+                    .field("format", "RGBA")
+                    .build(),
+            );
         let video_sink = videoconvert.link(&appsink).change_context(Error)?;
         let playbin = gst::plugins::playback::Playbin3::new("iced-video")
             .change_context(Error)?
             .with_uri(url.as_ref())
             .with_video_sink(&video_sink);
         let bus = playbin.bus().change_context(Error)?;
-        playbin.wait_ready()?;
-        // let bus_stream = bus.stream();
-        // bus_stream.find(|message| {
-        //     let view = message.view();
-        //     if let gst::MessageView::StateChanged(change) = view {
-        //         change.current() == gst::State::Ready
-        //     } else {
-        //         false
-        //     }
-        // });
+        playbin.pause().change_context(Error)?;
+        let ready = Arc::new(AtomicBool::new(false));
+        let frame = Arc::new(Mutex::new(Vec::new()));
+
+        let appsink = appsink.on_new_frame({
+            let ready = Arc::clone(&ready);
+            let frame = Arc::clone(&frame);
+            move |appsink| {
+                let Ok(sample) = appsink.pull_sample() else {
+                    return Ok(());
+                };
+                let caps = sample.caps().ok_or(gst::gstreamer::FlowError::Error)?;
+                let structure_0 = caps.structure(0).ok_or(gst::gstreamer::FlowError::Error)?;
+                let width = structure_0
+                    .get::<i32>("width")
+                    .map_err(|_| gst::gstreamer::FlowError::Error)?;
+                let height = structure_0
+                    .get::<i32>("height")
+                    .map_err(|_| gst::gstreamer::FlowError::Error)?;
+
+                let buffer = sample.buffer().and_then(|b| b.map_readable().ok());
+                if let Some(buffer) = buffer {
+                    {
+                        let mut frame = frame.lock().expect("BUG: Mutex poisoned");
+                        debug_assert_eq!(buffer.size(), (width * height * 4) as usize);
+                        if frame.len() != buffer.size() {
+                            frame.resize(buffer.size(), 0);
+                        }
+                        frame.copy_from_slice(buffer.as_slice());
+                        ready.store(true, std::sync::atomic::Ordering::Relaxed);
+                    }
+                    // if written.is_err() {
+                    //     tracing::error!("Failed to write video frame to buffer");
+                    // } else {
+                    //     ready.store(true, std::sync::atomic::Ordering::Relaxed);
+                    // }
+                }
+                Ok(())
+            }
+        });
 
         Ok(Self {
             playbin,
             videoconvert,
             appsink,
             bus,
+            ready,
+            frame,
         })
     }
 
+    pub async fn wait(self) -> Result<()> {
+        self.playbin
+            .wait_for_states(&[gst::State::Paused, gst::State::Playing])
+            .await
+            .change_context(Error)
+            .attach("Failed to wait for video initialisation")
+    }
+
     pub fn play(&self) -> Result<()> {
         self.playbin
             .play()
@@ -53,12 +116,15 @@ impl VideoSource {
             .attach("Failed to pause video")
     }
 
-    pub fn bus(&self) -> &Bus {}
-    // pub fn copy_frame_to_texture(&self, texture: wgpu::TextureView) -> Result<()> {
-    //     let frame = self
-    //         .appsink
-    //         .try_pull_sample(core::time::Duration::from_millis(1))?
-    //         .ok_or(Error)
-    //         .attach("No video frame available")?;
-    // }
+    pub fn size(&self) -> Result<(i32, i32)> {
+        let caps = self
+            .appsink
+            .sink("sink")
+            .current_caps()
+            .change_context(Error)?;
+        caps.width()
+            .and_then(|width| caps.height().map(|height| (width, height)))
+            .ok_or(Error)
+            .attach("Failed to get width, height")
+    }
 }