feat: Get iced-video working

Cargo.lock

@@ -3051,14 +3051,15 @@ version = "0.1.0"
 dependencies = [
  "error-stack",
  "futures",
+ "futures-lite 2.6.1",
  "glib 0.21.5",
  "gstreamer 0.24.4",
  "gstreamer-app 0.24.4",
  "gstreamer-video 0.24.4",
+ "smol",
  "thiserror 2.0.17",
  "tracing",
  "tracing-subscriber",
- "wgpu",
 ]
 
 [[package]]
@@ -3614,10 +3615,14 @@ version = "0.1.0"
 dependencies = [
  "error-stack",
  "gst",
+ "iced",
  "iced_core",
+ "iced_futures",
+ "iced_renderer",
  "iced_wgpu",
  "thiserror 2.0.17",
  "tracing",
+ "tracing-subscriber",
 ]
 
 [[package]]
@@ -8085,6 +8090,8 @@ dependencies = [
  "gpui_util",
  "iced",
  "iced_video_player",
+ "iced_wgpu",
+ "iced_winit",
  "reqwest",
  "tap",
  "toml 0.9.8",

crates/iced-video/Cargo.toml

@@ -7,6 +7,12 @@ edition = "2024"
 error-stack = "0.6.0"
 gst.workspace = true
 iced_core = "0.14.0"
+iced_futures = "0.14.0"
+iced_renderer = { version = "0.14.0", features = ["iced_wgpu"] }
 iced_wgpu = "0.14.0"
 thiserror = "2.0.17"
 tracing = "0.1.43"
+
+[dev-dependencies]
+iced.workspace = true
+tracing-subscriber = { version = "0.3.22", features = ["env-filter"] }

crates/iced-video/src/lib.rs

@@ -1,9 +1,12 @@
+pub mod id;
 pub mod primitive;
 pub mod source;
+use iced_core as iced;
+use iced_renderer::Renderer as RendererWithFallback;
+use iced_wgpu::primitive::Renderer as PrimitiveRenderer;
 
 use error_stack::{Report, ResultExt};
-use gst::*;
+use iced::Length;
-use iced_core::Length;
 use std::marker::PhantomData;
 
 use gst::plugins::app::AppSink;
@@ -16,16 +19,21 @@ pub struct Error;
 pub type Result<T, E = Report<Error>> = core::result::Result<T, E>;
 
 use std::sync::{Arc, Mutex, atomic::AtomicBool};
-pub struct Video {
+
-    id: iced_core::Id,
+/// This is the video handle that is used to control the video playback.
+/// This should be keps in the application state.
+#[derive(Debug, Clone)]
+pub struct VideoHandle {
+    id: id::Id,
     source: source::VideoSource,
+    is_metadata_loaded: Arc<AtomicBool>,
     is_playing: Arc<AtomicBool>,
     is_eos: Arc<AtomicBool>,
-    texture: Mutex<Option<iced_wgpu::wgpu::TextureView>>,
+    frame_ready: Arc<AtomicBool>,
 }
 
-impl Video {
+impl VideoHandle {
-    pub fn id(&self) -> &iced_core::Id {
+    pub fn id(&self) -> &id::Id {
         &self.id
     }
 
@@ -33,91 +41,212 @@ impl Video {
         &self.source
     }
 
-    pub async fn new(url: impl AsRef<str>) -> Result<Self> {
+    pub fn new(url: impl AsRef<str>) -> Result<Self> {
+        let source = source::VideoSource::new(url)?;
+        let frame_ready = Arc::clone(&source.ready);
         Ok(Self {
-            id: iced_core::Id::unique(),
+            id: id::Id::unique(),
-            source: source::VideoSource::new(url)?,
+            source: source,
+            is_metadata_loaded: Arc::new(AtomicBool::new(false)),
             is_playing: Arc::new(AtomicBool::new(false)),
             is_eos: Arc::new(AtomicBool::new(false)),
-            texture: Mutex::new(None),
+            frame_ready,
         })
     }
 }
 
-pub struct VideoPlayer<'a, Message, Theme = iced_core::Theme, Renderer = iced_wgpu::Renderer>
+/// This is the Video widget that displays a video.
+/// This should be used in the view function.
+pub struct Video<'a, Message, Theme = iced::Theme, Renderer = iced_wgpu::Renderer>
 where
     Renderer: PrimitiveRenderer,
 {
-    videos: &'a Video,
+    id: id::Id,
-    content_fit: iced_core::ContentFit,
+    handle: &'a VideoHandle,
-    width: iced_core::Length,
+    content_fit: iced::ContentFit,
-    height: iced_core::Length,
+    width: iced::Length,
+    height: iced::Length,
     on_end_of_stream: Option<Message>,
     on_new_frame: Option<Message>,
     looping: bool,
     // on_subtitle_text: Option<Box<dyn Fn(Option<String>) -> Message + 'a>>,
     // on_error: Option<Box<dyn Fn(&glib::Error) -> Message + 'a>>,
-    theme: Theme,
+    // theme: Theme,
-    __marker: PhantomData<Renderer>,
+    __marker: PhantomData<(Renderer, Theme)>,
 }
 
-impl<Message, Theme, Renderer> VideoPlayer<Message, Theme, Renderer>
+impl<'a, Message, Theme, Renderer> Video<'a, Message, Theme, Renderer>
 where
     Renderer: PrimitiveRenderer,
 {
-    pub fn new(source: source::VideoSource) -> Self {
+    pub fn new(handle: &'a VideoHandle) -> Self {
         Self {
-            videos: Video {
+            id: handle.id.clone(),
-                id: iced_core::Id::unique(),
+            handle: &handle,
-                source,
+            content_fit: iced::ContentFit::Contain,
-                is_playing: Arc::new(AtomicBool::new(false)),
-                is_eos: Arc::new(AtomicBool::new(false)),
-                texture: Mutex::new(None),
-            },
-            content_fit: iced_core::ContentFit::Contain,
             width: Length::Shrink,
             height: Length::Shrink,
             on_end_of_stream: None,
             on_new_frame: None,
             looping: false,
-            theme: Theme::default(),
+            // theme: Theme::default(),
             __marker: PhantomData,
         }
     }
 }
 
-impl<Message, Theme, Renderer> iced_core::Widget<Message, Theme, Renderer>
+impl<'a, Message, Theme, Renderer> Video<'a, Message, Theme, Renderer>
-    for VideoPlayer<'_, Message, Theme, Renderer>
+where
+    Renderer: PrimitiveRenderer,
+{
+    pub fn width(mut self, width: Length) -> Self {
+        self.width = width;
+        self
+    }
+
+    pub fn height(mut self, height: Length) -> Self {
+        self.height = height;
+        self
+    }
+
+    pub fn content_fit(mut self, fit: iced::ContentFit) -> Self {
+        self.content_fit = fit;
+        self
+    }
+
+    pub fn on_end_of_stream(mut self, message: Message) -> Self {
+        self.on_end_of_stream = Some(message);
+        self
+    }
+
+    pub fn on_new_frame(mut self, message: Message) -> Self {
+        self.on_new_frame = Some(message);
+        self
+    }
+
+    pub fn looping(mut self, looping: bool) -> Self {
+        self.looping = looping;
+        self
+    }
+}
+
+impl<Message, Theme, Renderer> iced::Widget<Message, Theme, Renderer>
+    for Video<'_, Message, Theme, Renderer>
 where
     Message: Clone,
     Renderer: PrimitiveRenderer,
 {
-    fn size(&self) -> iced_core::Size<Length> {
+    fn size(&self) -> iced::Size<Length> {
-        iced_core::Size {
+        iced::Size {
             width: self.width,
             height: self.height,
         }
     }
 
+    // The video player should take max space by default
     fn layout(
         &mut self,
-        iced_core::widget::tree: &mut iced_core::widget::Tree,
+        _tree: &mut iced::widget::Tree,
-        iced_core::renderer: &Renderer,
+        _renderer: &Renderer,
-        limits: &iced_core::layout::Limits,
+        limits: &iced::layout::Limits,
-    ) -> iced_core::layout::Node {
+    ) -> iced::layout::Node {
-        todo!()
+        iced::layout::Node::new(limits.max())
     }
 
     fn draw(
         &self,
-        iced_core::widget::tree: &iced_core::widget::Tree,
+        tree: &iced::widget::Tree,
-        iced_core::renderer: &mut Renderer,
+        renderer: &mut Renderer,
         theme: &Theme,
-        style: &iced_core::renderer::Style,
+        style: &iced::renderer::Style,
-        iced_core::layout: iced_core::Layout<'_>,
+        layout: iced::Layout<'_>,
-        cursor: iced_core::mouse::Cursor,
+        cursor: iced::mouse::Cursor,
-        viewport: &iced_core::Rectangle,
+        viewport: &iced::Rectangle,
     ) {
-        todo!()
+        if let Ok((width, height)) = self.handle.source.size() {
+            let video_size = iced::Size {
+                width: width as f32,
+                height: height as f32,
+            };
+            let bounds = layout.bounds();
+            let adjusted_fit = self.content_fit.fit(video_size, bounds.size());
+            let scale = iced::Vector::new(
+                adjusted_fit.width / video_size.width,
+                adjusted_fit.height / video_size.height,
+            );
+            let final_size = video_size * scale;
+            let position = match self.content_fit {
+                iced::ContentFit::None => iced::Point::new(
+                    bounds.x + (video_size.width - adjusted_fit.width) / 2.0,
+                    bounds.y + (video_size.height - adjusted_fit.height) / 2.0,
+                ),
+                _ => iced::Point::new(
+                    bounds.center_x() - final_size.width / 2.0,
+                    bounds.center_y() - final_size.height / 2.0,
+                ),
+            };
+
+            let drawing_bounds = iced::Rectangle::new(position, final_size);
+
+            let render = |renderer: &mut Renderer| {
+                renderer.draw_primitive(
+                    drawing_bounds,
+                    primitive::VideoFrame {
+                        id: self.id.clone(),
+                        size: iced_wgpu::wgpu::Extent3d {
+                            width: width as u32,
+                            height: height as u32,
+                            depth_or_array_layers: 1,
+                        },
+                        ready: Arc::clone(&self.handle.frame_ready),
+                        frame: Arc::clone(&self.handle.source.frame),
+                    },
+                );
+            };
+
+            if adjusted_fit.width > bounds.width || adjusted_fit.height > bounds.height {
+                renderer.with_layer(bounds, render);
+            } else {
+                render(renderer);
+            }
+        }
+    }
+
+    fn update(
+        &mut self,
+        _tree: &mut iced_core::widget::Tree,
+        event: &iced::Event,
+        _layout: iced_core::Layout<'_>,
+        _cursor: iced_core::mouse::Cursor,
+        _renderer: &Renderer,
+        _clipboard: &mut dyn iced_core::Clipboard,
+        shell: &mut iced_core::Shell<'_, Message>,
+        _viewport: &iced::Rectangle,
+    ) {
+        if let iced::Event::Window(iced::window::Event::RedrawRequested(_)) = event {
+            if self
+                .handle
+                .frame_ready
+                .load(std::sync::atomic::Ordering::SeqCst)
+            {
+                shell.request_redraw();
+            } else {
+                shell.request_redraw_at(iced::window::RedrawRequest::At(
+                    iced_core::time::Instant::now() + core::time::Duration::from_millis(32),
+                ));
+            }
+        }
+    }
+}
+
+impl<'a, Message, Theme, Renderer> From<Video<'a, Message, Theme, Renderer>>
+    for iced::Element<'a, Message, Theme, Renderer>
+where
+    Message: 'a + Clone,
+    Theme: 'a,
+    Renderer: 'a + iced_wgpu::primitive::Renderer,
+{
+    fn from(video: Video<'a, Message, Theme, Renderer>) -> Self {
+        Self::new(video)
     }
 }

crates/iced-video/src/primitive.rs

@@ -1,14 +1,18 @@
+use crate::id;
 use iced_wgpu::primitive::Pipeline;
 use iced_wgpu::wgpu;
 use std::collections::BTreeMap;
-use std::sync::{Arc, atomic::AtomicBool};
+use std::sync::{Arc, Mutex, atomic::AtomicBool};
 
 #[derive(Debug)]
-pub struct VideoPrimitive {
+pub struct VideoFrame {
-    texture: wgpu::TextureView,
+    pub id: id::Id,
-    ready: Arc<AtomicBool>,
+    pub size: wgpu::Extent3d,
+    pub ready: Arc<AtomicBool>,
+    pub frame: Arc<Mutex<Vec<u8>>>,
 }
-impl iced_wgpu::Primitive for VideoPrimitive {
+
+impl iced_wgpu::Primitive for VideoFrame {
     type Pipeline = VideoPipeline;
 
     fn prepare(
@@ -19,11 +23,102 @@ impl iced_wgpu::Primitive for VideoPrimitive {
         bounds: &iced_wgpu::core::Rectangle,
         viewport: &iced_wgpu::graphics::Viewport,
     ) {
-        todo!()
+        let video = pipeline.videos.entry(self.id.clone()).or_insert_with(|| {
+            let texture = device.create_texture(&wgpu::TextureDescriptor {
+                label: Some("iced-video-texture"),
+                size: self.size,
+                mip_level_count: 1,
+                sample_count: 1,
+                dimension: wgpu::TextureDimension::D2,
+                format: pipeline.format,
+                usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
+                view_formats: &[],
+            });
+            let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor {
+                label: Some("iced-video-texture-bind-group"),
+                layout: &pipeline.bind_group_layout,
+                entries: &[
+                    wgpu::BindGroupEntry {
+                        binding: 0,
+                        resource: wgpu::BindingResource::TextureView(
+                            &texture.create_view(&wgpu::TextureViewDescriptor::default()),
+                        ),
+                    },
+                    wgpu::BindGroupEntry {
+                        binding: 1,
+                        resource: wgpu::BindingResource::Sampler(&pipeline.sampler),
+                    },
+                ],
+            });
+            VideoTextures {
+                id: self.id.clone(),
+                texture,
+                bind_group,
+                ready: Arc::clone(&self.ready),
+            }
+        });
+        // dbg!(&self.size, video.texture.size());
+        if self.size != video.texture.size() {
+            // Resize the texture if the size has changed.
+            let new_texture = device.create_texture(&wgpu::TextureDescriptor {
+                label: Some("iced-video-texture-resized"),
+                size: self.size,
+                mip_level_count: 1,
+                sample_count: 1,
+                dimension: wgpu::TextureDimension::D2,
+                format: pipeline.format,
+                usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
+                view_formats: &[],
+            });
+            let new_bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor {
+                label: Some("iced-video-texture-bind-group-resized"),
+                layout: &pipeline.bind_group_layout,
+                entries: &[
+                    wgpu::BindGroupEntry {
+                        binding: 0,
+                        resource: wgpu::BindingResource::TextureView(
+                            &new_texture.create_view(&wgpu::TextureViewDescriptor::default()),
+                        ),
+                    },
+                    wgpu::BindGroupEntry {
+                        binding: 1,
+                        resource: wgpu::BindingResource::Sampler(&pipeline.sampler),
+                    },
+                ],
+            });
+            video.texture = new_texture;
+            video.bind_group = new_bind_group;
+        }
+        // BUG: This causes a panic because the texture size is not correct for some reason.
+        if video.ready.load(std::sync::atomic::Ordering::SeqCst) {
+            let frame = self.frame.lock().expect("BUG: Mutex poisoned");
+            if frame.len() != (4 * self.size.width * self.size.height) as usize {
+                tracing::warn!(
+                    "Frame size mismatch: expected {}, got {}",
+                    4 * self.size.width * self.size.height,
+                    frame.len()
+                );
+                return;
+            }
+            queue.write_texture(
+                wgpu::TexelCopyTextureInfo {
+                    texture: &video.texture,
+                    mip_level: 0,
+                    origin: wgpu::Origin3d::ZERO,
+                    aspect: wgpu::TextureAspect::All,
+                },
+                &frame,
+                wgpu::TexelCopyBufferLayout {
+                    offset: 0,
+                    bytes_per_row: Some(4 * video.texture.size().width),
+                    rows_per_image: Some(video.texture.size().height),
+                },
+                self.size,
+            );
+            video
+                .ready
+                .store(false, std::sync::atomic::Ordering::SeqCst);
         }
-
-    fn draw(&self, _pipeline: &Self::Pipeline, _render_pass: &mut wgpu::RenderPass<'_>) -> bool {
-        false
     }
 
     fn render(
@@ -33,14 +128,21 @@ impl iced_wgpu::Primitive for VideoPrimitive {
         target: &wgpu::TextureView,
         clip_bounds: &iced_wgpu::core::Rectangle<u32>,
     ) {
-        if self.ready.load(std::sync::atomic::Ordering::SeqCst) {
+        let Some(video) = pipeline.videos.get(&self.id) else {
+            return;
+        };
         let mut render_pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor {
             label: Some("iced-video-render-pass"),
             color_attachments: &[Some(wgpu::RenderPassColorAttachment {
                 view: target,
                 resolve_target: None,
                 ops: wgpu::Operations {
-                        load: wgpu::LoadOp::Load,
+                    load: wgpu::LoadOp::Clear(wgpu::Color {
+                        r: 0.1,
+                        g: 0.2,
+                        b: 0.3,
+                        a: 1.0,
+                    }),
                     store: wgpu::StoreOp::Store,
                 },
                 depth_slice: None,
@@ -51,17 +153,16 @@ impl iced_wgpu::Primitive for VideoPrimitive {
         });
 
         render_pass.set_pipeline(&pipeline.pipeline);
-            render_pass.set_bind_group(0, &self.bind_group, &[]);
+        render_pass.set_bind_group(0, &video.bind_group, &[]);
         render_pass.draw(0..3, 0..1);
-            self.ready
+        // self.ready
-                .store(false, std::sync::atomic::Ordering::Relaxed);
+        //     .store(false, std::sync::atomic::Ordering::Relaxed);
-        }
     }
 }
 
 #[derive(Debug)]
 pub struct VideoTextures {
-    id: u64,
+    id: id::Id,
     texture: wgpu::Texture,
     bind_group: wgpu::BindGroup,
     ready: Arc<AtomicBool>,
@@ -72,7 +173,8 @@ pub struct VideoPipeline {
     pipeline: wgpu::RenderPipeline,
     bind_group_layout: wgpu::BindGroupLayout,
     sampler: wgpu::Sampler,
-    videos: BTreeMap<u64, VideoTextures>,
+    videos: BTreeMap<id::Id, VideoTextures>,
+    format: wgpu::TextureFormat,
 }
 
 pub trait HdrTextureFormatExt {
@@ -171,6 +273,7 @@ impl Pipeline for VideoPipeline {
             pipeline,
             bind_group_layout,
             sampler,
+            format,
             videos: BTreeMap::new(),
         }
     }

crates/iced-video/src/source.rs

@@ -1,44 +1,107 @@
+use crate::{Error, Result, ResultExt};
+use gst::{
+    Bus, Gst, MessageType, MessageView, Sink, Source,
+    app::AppSink,
+    caps::{Caps, CapsType},
+    element::ElementExt,
+    pipeline::PipelineExt,
+    playback::Playbin3,
+    videoconvertscale::VideoConvert,
+};
+use std::sync::{Arc, Mutex, atomic::AtomicBool};
+
 #[derive(Debug, Clone)]
 pub struct VideoSource {
-    playbin: Playbin3,
+    pub(crate) playbin: Playbin3,
-    videoconvert: VideoConvert,
+    pub(crate) videoconvert: VideoConvert,
-    appsink: AppSink,
+    pub(crate) appsink: AppSink,
-    bus: Bus,
+    pub(crate) bus: Bus,
+    pub(crate) ready: Arc<AtomicBool>,
+    pub(crate) frame: Arc<Mutex<Vec<u8>>>,
 }
 
 impl VideoSource {
     /// Creates a new video source from the given URL.
     /// Since this doesn't have to parse the pipeline manually, we aren't sanitizing the URL for
     /// now.
-    pub async fn new(url: impl AsRef<str>) -> Result<Self> {
+    pub fn new(url: impl AsRef<str>) -> Result<Self> {
         Gst::new();
-        let videoconvert = VideoConvert::new("iced-video-convert").change_context(Error)?;
+        let videoconvert = VideoConvert::new("iced-video-convert")
-        let appsink = AppSink::new("iced-video-sink").change_context(Error)?;
+            // .change_context(Error)?
+            // .with_output_format(gst::plugins::videoconvertscale::VideoFormat::Rgba)
+            .change_context(Error)?;
+        let appsink = AppSink::new("iced-video-sink")
+            .change_context(Error)?
+            .with_caps(
+                Caps::builder(CapsType::Video)
+                    .field("format", "RGBA")
+                    .build(),
+            );
         let video_sink = videoconvert.link(&appsink).change_context(Error)?;
         let playbin = gst::plugins::playback::Playbin3::new("iced-video")
             .change_context(Error)?
             .with_uri(url.as_ref())
             .with_video_sink(&video_sink);
         let bus = playbin.bus().change_context(Error)?;
-        playbin.wait_ready()?;
+        playbin.pause().change_context(Error)?;
-        // let bus_stream = bus.stream();
+        let ready = Arc::new(AtomicBool::new(false));
-        // bus_stream.find(|message| {
+        let frame = Arc::new(Mutex::new(Vec::new()));
-        //     let view = message.view();
+
-        //     if let gst::MessageView::StateChanged(change) = view {
+        let appsink = appsink.on_new_frame({
-        //         change.current() == gst::State::Ready
+            let ready = Arc::clone(&ready);
+            let frame = Arc::clone(&frame);
+            move |appsink| {
+                let Ok(sample) = appsink.pull_sample() else {
+                    return Ok(());
+                };
+                let caps = sample.caps().ok_or(gst::gstreamer::FlowError::Error)?;
+                let structure_0 = caps.structure(0).ok_or(gst::gstreamer::FlowError::Error)?;
+                let width = structure_0
+                    .get::<i32>("width")
+                    .map_err(|_| gst::gstreamer::FlowError::Error)?;
+                let height = structure_0
+                    .get::<i32>("height")
+                    .map_err(|_| gst::gstreamer::FlowError::Error)?;
+
+                let buffer = sample.buffer().and_then(|b| b.map_readable().ok());
+                if let Some(buffer) = buffer {
+                    {
+                        let mut frame = frame.lock().expect("BUG: Mutex poisoned");
+                        debug_assert_eq!(buffer.size(), (width * height * 4) as usize);
+                        if frame.len() != buffer.size() {
+                            frame.resize(buffer.size(), 0);
+                        }
+                        frame.copy_from_slice(buffer.as_slice());
+                        ready.store(true, std::sync::atomic::Ordering::Relaxed);
+                    }
+                    // if written.is_err() {
+                    //     tracing::error!("Failed to write video frame to buffer");
                     // } else {
-        //         false
+                    //     ready.store(true, std::sync::atomic::Ordering::Relaxed);
                     // }
-        // });
+                }
+                Ok(())
+            }
+        });
 
         Ok(Self {
             playbin,
             videoconvert,
             appsink,
             bus,
+            ready,
+            frame,
         })
     }
 
+    pub async fn wait(self) -> Result<()> {
+        self.playbin
+            .wait_for_states(&[gst::State::Paused, gst::State::Playing])
+            .await
+            .change_context(Error)
+            .attach("Failed to wait for video initialisation")
+    }
+
     pub fn play(&self) -> Result<()> {
         self.playbin
             .play()
@@ -53,12 +116,15 @@ impl VideoSource {
             .attach("Failed to pause video")
     }
 
-    pub fn bus(&self) -> &Bus {}
+    pub fn size(&self) -> Result<(i32, i32)> {
-    // pub fn copy_frame_to_texture(&self, texture: wgpu::TextureView) -> Result<()> {
+        let caps = self
-    //     let frame = self
+            .appsink
-    //         .appsink
+            .sink("sink")
-    //         .try_pull_sample(core::time::Duration::from_millis(1))?
+            .current_caps()
-    //         .ok_or(Error)
+            .change_context(Error)?;
-    //         .attach("No video frame available")?;
+        caps.width()
-    // }
+            .and_then(|width| caps.height().map(|height| (width, height)))
+            .ok_or(Error)
+            .attach("Failed to get width, height")
+    }
 }

flake.lock

@@ -3,11 +3,11 @@
     "advisory-db": {
       "flake": false,
       "locked": {
-        "lastModified": 1765811277,
+        "lastModified": 1766435619,
-        "narHash": "sha256-QF/aUvQwJG/ndoRZCjb+d7xASs0ELCmpqpK8u6Se2f4=",
+        "narHash": "sha256-3A5Z5K28YB45REOHMWtyQ24cEUXW76MOtbT6abPrARE=",
         "owner": "rustsec",
         "repo": "advisory-db",
-        "rev": "2d254c1fad2260522209e9bce2fdc93012b0627f",
+        "rev": "a98dbc80b16730a64c612c6ab5d5fecb4ebb79ba",
         "type": "github"
       },
       "original": {
@@ -18,11 +18,11 @@
     },
     "crane": {
       "locked": {
-        "lastModified": 1765739568,
+        "lastModified": 1766194365,
-        "narHash": "sha256-gQYx35Of4UDKUjAYvmxjUEh/DdszYeTtT6MDin4loGE=",
+        "narHash": "sha256-4AFsUZ0kl6MXSm4BaQgItD0VGlEKR3iq7gIaL7TjBvc=",
         "owner": "ipetkov",
         "repo": "crane",
-        "rev": "67d2baff0f9f677af35db61b32b5df6863bcc075",
+        "rev": "7d8ec2c71771937ab99790b45e6d9b93d15d9379",
         "type": "github"
       },
       "original": {
@@ -106,11 +106,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1765779637,
+        "lastModified": 1766309749,
-        "narHash": "sha256-KJ2wa/BLSrTqDjbfyNx70ov/HdgNBCBBSQP3BIzKnv4=",
+        "narHash": "sha256-3xY8CZ4rSnQ0NqGhMKAy5vgC+2IVK0NoVEzDoOh4DA4=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "1306659b587dc277866c7b69eb97e5f07864d8c4",
+        "rev": "a6531044f6d0bef691ea18d4d4ce44d0daa6e816",
         "type": "github"
       },
       "original": {
@@ -138,11 +138,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1765852971,
+        "lastModified": 1766371695,
-        "narHash": "sha256-rQdOMqfQNhcfqvh1dFIVWh09mrIWwerUJqqBdhIsf8g=",
+        "narHash": "sha256-W7CX9vy7H2Jj3E8NI4djHyF8iHSxKpb2c/7uNQ/vGFU=",
         "owner": "oxalica",
         "repo": "rust-overlay",
-        "rev": "5f98ccecc9f1bc1c19c0a350a659af1a04b3b319",
+        "rev": "d81285ba8199b00dc31847258cae3c655b605e8c",
         "type": "github"
       },
       "original": {

flake.nix

@@ -195,12 +195,13 @@
                     cargo-outdated
                     lld
                     lldb
-                    cargo-flamegraph
+                    cargo-audit
                   ]
                   ++ (lib.optionals pkgs.stdenv.isDarwin [
                     apple-sdk_26
                   ])
                   ++ (lib.optionals pkgs.stdenv.isLinux [
+                    cargo-flamegraph
                     perf
                     mold
                   ]);

gst/Cargo.toml

@@ -8,13 +8,14 @@ edition = "2024"
 [dependencies]
 error-stack = "0.6"
 futures = "0.3.31"
+futures-lite = "2.6.1"
 glib = "0.21.5"
 gstreamer = { version = "0.24.4", features = ["v1_18"] }
 gstreamer-app = { version = "0.24.4", features = ["v1_18"] }
 gstreamer-video = { version = "0.24.4", features = ["v1_18"] }
 thiserror = "2.0"
 tracing = { version = "0.1", features = ["log"] }
-wgpu = { version = "27.0.1", default-features = false }
 
 [dev-dependencies]
+smol = "2.0.2"
 tracing-subscriber = "0.3.22"

gst/src/bus.rs

@@ -17,4 +17,11 @@ impl Bus {
     pub fn stream(&self) -> gstreamer::bus::BusStream {
         self.inner.stream()
     }
+
+    pub fn filtered_stream<'a>(
+        &self,
+        msg_types: &'a [gstreamer::MessageType],
+    ) -> impl futures::stream::FusedStream<Item = gstreamer::Message> + Unpin + Send + 'a {
+        self.inner.stream_filtered(msg_types)
+    }
 }

gst/src/caps.rs

@@ -1,4 +1,6 @@
-use crate::*;
+use gstreamer::Fraction;
+
+#[derive(Debug, Clone)]
 #[repr(transparent)]
 pub struct Caps {
     pub(crate) inner: gstreamer::caps::Caps,
@@ -16,7 +18,6 @@ pub struct CapsBuilder {
 
 impl CapsBuilder {
     pub fn field<V: Into<glib::Value> + Send>(mut self, name: impl AsRef<str>, value: V) -> Self {
-        use gstreamer::prelude::*;
         self.inner = self.inner.field(name.as_ref(), value);
         self
     }
@@ -53,10 +54,25 @@ impl CapsBuilder {
 }
 
 impl Caps {
-    pub fn format(&self) -> Option<&str> {
+    pub fn format(&self) -> Option<gstreamer_video::VideoFormat> {
-        use gstreamer::prelude::*;
         self.inner
             .structure(0)
             .and_then(|s| s.get::<&str>("format").ok())
+            .map(|s| gstreamer_video::VideoFormat::from_string(s))
+    }
+    pub fn width(&self) -> Option<i32> {
+        self.inner
+            .structure(0)
+            .and_then(|s| s.get::<i32>("width").ok())
+    }
+    pub fn height(&self) -> Option<i32> {
+        self.inner
+            .structure(0)
+            .and_then(|s| s.get::<i32>("height").ok())
+    }
+    pub fn framerate(&self) -> Option<gstreamer::Fraction> {
+        self.inner
+            .structure(0)
+            .and_then(|s| s.get::<Fraction>("framerate").ok())
     }
 }

gst/src/element.rs

@@ -27,6 +27,15 @@ impl Element {
         use gstreamer::prelude::*;
         self.inner.static_pad(name.as_ref()).map(Pad::from)
     }
+
+    pub fn bus(&self) -> Result<Bus> {
+        use gstreamer::prelude::*;
+        self.inner
+            .bus()
+            .map(Bus::from)
+            .ok_or(Error)
+            .attach_with(|| format!("Failed to get bus from Element: {}", self.inner.name()))
+    }
 }
 
 pub trait Sink: ChildOf<Element> {
@@ -108,3 +117,17 @@ pub trait Source: ChildOf<Element> {
     //     Ok(())
     // }
 }
+
+pub trait ElementExt: ChildOf<Element> + Sync {
+    #[track_caller]
+    fn bus(&self) -> Result<Bus> {
+        self.upcast_ref().bus()
+    }
+
+    #[track_caller]
+    fn pad(&self, name: impl AsRef<str>) -> Option<Pad> {
+        self.upcast_ref().pad(name)
+    }
+}
+
+impl<T: ChildOf<Element> + Sync> ElementExt for T {}

gst/src/lib.rs

@@ -13,6 +13,9 @@ pub use bin::*;
 pub use bus::*;
 pub use caps::*;
 pub use element::*;
+pub use gstreamer;
+#[doc(inline)]
+pub use gstreamer::{Message, MessageType, MessageView, State};
 pub use pad::*;
 pub use pipeline::*;
 pub use plugins::*;
@@ -21,6 +24,7 @@ pub(crate) mod priv_prelude {
     pub use crate::errors::*;
     pub use crate::wrapper::*;
     pub use crate::*;
+    pub use gstreamer::prelude::ElementExt as _;
     pub use gstreamer::prelude::*;
     #[track_caller]
     pub fn duration_to_clocktime(
@@ -33,15 +37,12 @@ pub(crate) mod priv_prelude {
                     .attach("Failed to convert duration to ClockTime")?;
                 Ok(Some(clocktime))
             }
-            None => Ok(None),
+            None => Ok(gstreamer::ClockTime::NONE),
         }
     }
 }
 
-use errors::*;
-use gstreamer::prelude::*;
 use std::sync::Arc;
-
 static GST: std::sync::LazyLock<std::sync::Arc<Gst>> = std::sync::LazyLock::new(|| {
     gstreamer::init().expect("Failed to initialize GStreamer");
     std::sync::Arc::new(Gst {
@@ -49,7 +50,6 @@ static GST: std::sync::LazyLock<std::sync::Arc<Gst>> = std::sync::LazyLock::new(
     })
 });
 
-/// This should be a global singleton
 pub struct Gst {
     __private: core::marker::PhantomData<()>,
 }
@@ -58,14 +58,4 @@ impl Gst {
     pub fn new() -> Arc<Self> {
         Arc::clone(&GST)
     }
-
-    // pub fn pipeline_from_str(&self, s: &str) -> Result<Pipeline> {
-    //     let pipeline = gstreamer::parse::launch(s).change_context(Error)?;
-    //     let pipeline = pipeline.downcast::<gstreamer::Pipeline>();
-    //     let pipeline = match pipeline {
-    //         Err(_e) => return Err(Error).attach("Failed to downcast to Pipeline"),
-    //         Ok(p) => p,
-    //     };
-    //     Ok(Pipeline { inner: pipeline })
-    // }
 }

gst/src/pad.rs

@@ -1,8 +1,9 @@
 use crate::priv_prelude::*;
-/// Pads are link points between elements
+
 wrap_gst!(Pad, gstreamer::Pad);
 
 impl Pad {
+    #[track_caller]
     pub fn ghost(target: &Pad) -> Result<Pad> {
         let ghost_pad = gstreamer::GhostPad::with_target(&target.inner)
             .change_context(Error)
@@ -12,6 +13,7 @@ impl Pad {
         })
     }
 
+    #[track_caller]
     pub fn link(&self, peer: &Pad) -> Result<()> {
         use gstreamer::prelude::*;
         self.inner
@@ -21,6 +23,7 @@ impl Pad {
         Ok(())
     }
 
+    #[track_caller]
     pub fn current_caps(&self) -> Result<Caps> {
         let caps = self
             .inner
@@ -30,6 +33,7 @@ impl Pad {
         Ok(Caps { inner: caps })
     }
 
+    #[track_caller]
     pub fn activate(&self, activate: bool) -> Result<()> {
         use gstreamer::prelude::*;
         self.inner

gst/src/pipeline.rs

@@ -12,6 +12,7 @@ impl Drop for Pipeline {
 }
 
 impl Pipeline {
+    #[track_caller]
     pub fn bus(&self) -> Result<Bus> {
         let bus = self
             .inner
@@ -22,15 +23,17 @@ impl Pipeline {
     }
 
     /// Get the state
+    #[track_caller]
     pub fn state(
         &self,
         timeout: impl Into<Option<core::time::Duration>>,
     ) -> Result<gstreamer::State> {
-        let (result, current, pending) = self.inner.state(duration_to_clocktime(timeout)?);
+        let (result, current, _pending) = self.inner.state(duration_to_clocktime(timeout)?);
         result.change_context(Error).attach("Failed to get state")?;
         Ok(current)
     }
 
+    #[track_caller]
     pub fn play(&self) -> Result<()> {
         self.inner
             .set_state(gstreamer::State::Playing)
@@ -39,6 +42,7 @@ impl Pipeline {
         Ok(())
     }
 
+    #[track_caller]
     pub fn pause(&self) -> Result<()> {
         self.inner
             .set_state(gstreamer::State::Paused)
@@ -47,6 +51,7 @@ impl Pipeline {
         Ok(())
     }
 
+    #[track_caller]
     pub fn ready(&self) -> Result<()> {
         self.inner
             .set_state(gstreamer::State::Ready)
@@ -55,6 +60,7 @@ impl Pipeline {
         Ok(())
     }
 
+    #[track_caller]
     pub fn set_state(&self, state: gstreamer::State) -> Result<gstreamer::StateChangeSuccess> {
         let result = self
             .inner
@@ -63,37 +69,135 @@ impl Pipeline {
             .attach("Failed to set pipeline state")?;
         Ok(result)
     }
+
+    pub async fn wait_for(&self, state: gstreamer::State) -> Result<()> {
+        let current_state = self.state(core::time::Duration::ZERO)?;
+        if current_state == state {
+            Ok(())
+        } else {
+            // use futures::stream::StreamExt;
+            use futures_lite::stream::StreamExt as _;
+            self.bus()?
+                .filtered_stream(&[MessageType::StateChanged])
+                .find(|message: &gstreamer::Message| {
+                    let view = message.view();
+                    if let gstreamer::MessageView::StateChanged(changed) = view {
+                        changed.current() == state
+                            && changed.src().is_some_and(|s| s == &self.inner)
+                    } else {
+                        false
+                    }
+                })
+                .await;
+            Ok(())
+        }
     }
 
-pub trait PipelineExt {
+    pub async fn wait_for_states(&self, states: impl AsRef<[gstreamer::State]>) -> Result<()> {
-    fn bus(&self) -> Result<Bus>;
+        let current_state = self.state(core::time::Duration::ZERO)?;
-    fn play(&self) -> Result<()>;
+        let states = states.as_ref();
-    fn pause(&self) -> Result<()>;
+        if states.contains(&current_state) {
-    fn ready(&self) -> Result<()>;
+            Ok(())
-    fn set_state(&self, state: gstreamer::State) -> Result<gstreamer::StateChangeSuccess>;
+        } else {
-    fn state(&self, timeout: impl Into<Option<core::time::Duration>>) -> Result<gstreamer::State>;
+            use futures_lite::stream::StreamExt as _;
+            self.bus()?
+                .filtered_stream(&[MessageType::StateChanged])
+                .find(|message: &gstreamer::Message| {
+                    let view = message.view();
+                    if let gstreamer::MessageView::StateChanged(changed) = view {
+                        states.contains(&changed.current())
+                            && changed.src().is_some_and(|s| s == &self.inner)
+                    } else {
+                        false
+                    }
+                })
+                .await;
+            Ok(())
+        }
     }
 
-impl<T> PipelineExt for T
+    pub async fn wait_for_message<'a, F2>(
+        &self,
+        filter: Option<&'a [gstreamer::MessageType]>,
+        filter_fn: F2,
+    ) -> Result<gstreamer::Message>
     where
-    T: ChildOf<Pipeline>,
+        F2: Fn(&gstreamer::Message) -> bool + Send + 'a,
     {
-    fn bus(&self) -> Result<Bus> {
+        use futures_lite::stream::StreamExt as _;
-        self.upcast_ref().bus()
+        match filter {
+            Some(filter) => {
+                let message = self.bus()?.filtered_stream(filter).find(filter_fn).await;
+                match message {
+                    Some(msg) => Ok(msg),
+                    None => {
+                        Err(Error).attach("Failed to find message matching the provided filter")
                     }
+                }
+            }
+            None => {
+                let message = self.bus()?.stream().find(filter_fn).await;
+                match message {
+                    Some(msg) => Ok(msg),
+                    None => {
+                        Err(Error).attach("Failed to find message matching the provided filter")
+                    }
+                }
+            }
+        }
+    }
+}
+
+pub trait PipelineExt: ChildOf<Pipeline> + Sync {
+    // #[track_caller]
+    // fn bus(&self) -> Result<Bus> {
+    //     self.upcast_ref().bus()
+    // }
+    #[track_caller]
     fn play(&self) -> Result<()> {
         self.upcast_ref().play()
     }
+    #[track_caller]
     fn pause(&self) -> Result<()> {
         self.upcast_ref().pause()
     }
+    #[track_caller]
     fn ready(&self) -> Result<()> {
         self.upcast_ref().ready()
     }
+    #[track_caller]
     fn set_state(&self, state: gstreamer::State) -> Result<gstreamer::StateChangeSuccess> {
         self.upcast_ref().set_state(state)
     }
+    #[track_caller]
     fn state(&self, timeout: impl Into<Option<core::time::Duration>>) -> Result<gstreamer::State> {
         self.upcast_ref().state(timeout)
     }
+
+    fn wait_for(
+        &self,
+        state: gstreamer::State,
+    ) -> impl std::future::Future<Output = Result<()>> + Send {
+        self.upcast_ref().wait_for(state)
     }
+
+    fn wait_for_states(
+        &self,
+        states: impl AsRef<[gstreamer::State]> + Send,
+    ) -> impl std::future::Future<Output = Result<()>> + Send {
+        self.upcast_ref().wait_for_states(states)
+    }
+
+    fn wait_for_message<'a, F2>(
+        &self,
+        filter: Option<&'a [gstreamer::MessageType]>,
+        filter_fn: F2,
+    ) -> impl std::future::Future<Output = Result<gstreamer::Message>> + Send
+    where
+        F2: Fn(&gstreamer::Message) -> bool + Send + 'a,
+    {
+        self.upcast_ref().wait_for_message(filter, filter_fn)
+    }
+}
+
+impl<T: ChildOf<Pipeline> + Sync> PipelineExt for T {}

gst/src/plugins/app/appsink.rs

@@ -1,7 +1,9 @@
 use crate::priv_prelude::*;
 
+#[doc(inline)]
+pub use gstreamer_app::AppSinkCallbacks;
+
 wrap_gst!(AppSink, gstreamer::Element);
-parent_child!(Pipeline, AppSink, downcast); // since AppSink is an Element internaly 
 parent_child!(Element, AppSink);
 
 impl Sink for AppSink {}
@@ -12,6 +14,7 @@ impl AppSink {
             .downcast_ref::<gstreamer_app::AppSink>()
             .expect("Failed to downcast to AppSink")
     }
+
     pub fn new(name: impl AsRef<str>) -> Result<Self> {
         use gstreamer::prelude::*;
         let inner = gstreamer::ElementFactory::make("appsink")
@@ -22,14 +25,47 @@ impl AppSink {
         Ok(AppSink { inner })
     }
 
-    pub fn with_caps(mut self, caps: Caps) -> Self {
+    pub fn with_emit_signals(self, emit: bool) -> Self {
+        self.inner.set_property("emit-signals", emit);
+        self
+    }
+
+    pub fn with_async(self, async_: bool) -> Self {
+        self.inner.set_property("async", async_);
+        self
+    }
+
+    pub fn with_sync(self, sync: bool) -> Self {
+        self.inner.set_property("sync", sync);
+        self
+    }
+
+    pub fn with_caps(self, caps: Caps) -> Self {
         self.inner.set_property("caps", caps.inner);
         self
     }
 
-    pub fn set_callbacks(&self, callbacks: gstreamer_app::AppSinkCallbacks) -> Result<()> {
+    pub fn with_callbacks(self, callbacks: gstreamer_app::AppSinkCallbacks) -> Self {
         self.appsink().set_callbacks(callbacks);
-        Ok(())
+        self
+    }
+
+    pub fn on_new_frame<F>(self, mut f: F) -> Self
+    where
+        F: FnMut(&AppSink) -> Result<(), gstreamer::FlowError> + Send + 'static,
+    {
+        self.with_emit_signals(true).with_callbacks(
+            AppSinkCallbacks::builder()
+                .new_sample(move |appsink| {
+                    use glib::object::Cast;
+                    let element = appsink.upcast_ref::<gstreamer::Element>();
+                    let appsink = AppSink::from_gst_ref(element);
+                    std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| f(appsink)))
+                        .unwrap_or(Err(gstreamer::FlowError::Error))
+                        .map(|_| gstreamer::FlowSuccess::Ok)
+                })
+                .build(),
+        )
     }
 
     pub fn pull_sample(&self) -> Result<Sample> {
@@ -151,3 +187,49 @@ fn test_appsink() {
     }
     // std::thread::sleep(std::time::Duration::from_secs(5));
 }
+
+#[test]
+fn test_appsink_metadata() {
+    use tracing_subscriber::prelude::*;
+    tracing_subscriber::registry()
+        .with(
+            tracing_subscriber::fmt::layer()
+                .with_thread_ids(true)
+                .with_file(true),
+        )
+        .init();
+
+    crate::Gst::new();
+
+    let url = "https://jellyfin.tsuba.darksailor.dev/Items/6010382cf25273e624d305907010d773/Download?api_key=036c140222464878862231ef66a2bc9c";
+
+    let videoconvert = crate::plugins::videoconvertscale::VideoConvert::new("iced-video-convert")
+        // .unwrap();
+        // .with_output_format(gst::plugins::videoconvertscale::VideoFormat::Rgba)
+        .unwrap();
+    let appsink = crate::plugins::app::AppSink::new("iced-video-sink")
+        .unwrap()
+        .with_async(true)
+        .with_sync(true);
+
+    let video_sink = videoconvert.link(&appsink).unwrap();
+    let playbin = crate::plugins::playback::Playbin3::new("iced-video")
+        .unwrap()
+        .with_uri(url)
+        .with_video_sink(&video_sink);
+
+    playbin.pause().unwrap();
+
+    smol::block_on(async {
+        playbin.wait_for(gstreamer::State::Paused).await.unwrap();
+    });
+    // std::thread::sleep(core::time::Duration::from_secs(1));
+    let pad = appsink.pad("sink").unwrap();
+    let caps = pad.current_caps().unwrap();
+    let format = caps.format();
+    let height = caps.height();
+    let width = caps.width();
+    let framerate = caps.framerate();
+    dbg!(&format, height, width, framerate);
+    dbg!(&caps);
+}

gst/src/wrapper.rs

@@ -3,6 +3,7 @@ pub trait GstWrapper {
     fn from_gst(gst: Self::GstType) -> Self;
     // fn into_gst(self) -> Self::GstType;
     fn as_gst_ref(&self) -> &Self::GstType;
+    fn from_gst_ref(gst: &Self::GstType) -> &Self;
 }
 
 #[macro_export]
@@ -51,6 +52,10 @@ macro_rules! wrap_gst {
             fn as_gst_ref(&self) -> &Self::GstType {
                 &self.inner
             }
+
+            fn from_gst_ref(gst: &Self::GstType) -> &Self {
+                unsafe { &*(gst as *const Self::GstType as *const Self) }
+            }
         }
 
         impl ChildOf<$name> for $name {
@@ -97,7 +102,14 @@ macro_rules! parent_child {
                 let downcasted = self
                     .inner
                     .downcast_ref::<<$parent as GstWrapper>::GstType>()
-                    .expect("BUG: Failed to downcast GStreamer type from child to parent");
+                    .expect(
+                        format!(
+                            "BUG: Failed to downcast GStreamer type from child {} to parent {}",
+                            stringify!($child),
+                            stringify!($parent)
+                        )
+                        .as_str(),
+                    );
                 unsafe {
                     &*(downcasted as *const <$parent as GstWrapper>::GstType as *const $parent)
                 }

src/main.rs

@@ -14,31 +14,3 @@ fn main() -> Result<()> {
     ui_iced::ui().change_context(Error)?;
     Ok(())
 }
-
-// #[tokio::main]
-// pub async fn main() -> Result<()> {
-//     dotenvy::dotenv()
-//         .change_context(Error)
-//         .inspect_err(|err| {
-//             eprintln!("Failed to load .env file: {}", err);
-//         })
-//         .ok();
-//     let config = JellyfinConfig::new(
-//         std::env::var("JELLYFIN_USERNAME").change_context(Error)?,
-//         std::env::var("JELLYFIN_PASSWORD").change_context(Error)?,
-//         std::env::var("JELLYFIN_SERVER_URL").change_context(Error)?,
-//         "jello".to_string(),
-//     );
-//     let mut jellyfin = api::JellyfinClient::new(config);
-//     jellyfin
-//         .authenticate_with_cached_token(".session")
-//         .await
-//         .change_context(Error)?;
-//
-//     #[cfg(feature = "iced")]
-//     ui_iced::ui(jellyfin);
-//     #[cfg(feature = "gpui")]
-//     ui_gpui::ui(jellyfin);
-//
-//     Ok(())
-// }

ui-iced/Cargo.toml

@@ -20,6 +20,8 @@ iced = { workspace = true, default-features = true, features = [
 
 
 iced_video_player = { workspace = true }
+iced_wgpu = "0.14.0"
+iced_winit = "0.14.0"
 reqwest = "0.12.24"
 tap = "1.0.1"
 toml = "0.9.8"