feat(gst): enhance GStreamer integration with new modules and improved API
Some checks failed
build / checks-matrix (push) Has been cancelled
build / codecov (push) Has been cancelled
docs / docs (push) Has been cancelled
build / checks-build (push) Has been cancelled

This commit introduces significant enhancements to the GStreamer integration by:
- Adding new modules for bins, caps, elements, pads, and plugins
- Implementing a more ergonomic API with helper methods like play(), pause(), ready()
- Adding support for various GStreamer plugins including app, autodetect, playback, and videoconvertscale
- Improving error handling with better context attachment
- Updating dependencies to latest versions including gstreamer-video 0.24.4
- Refactoring existing code to use modern Rust patterns and features
This commit is contained in:
uttarayan21
2025-12-17 23:35:05 +05:30
parent 21cbaff610
commit d42ef3b550
19 changed files with 619 additions and 117 deletions

51
gst/src/bin.rs Normal file
View File

@@ -0,0 +1,51 @@
use crate::*;
pub struct Bin {
inner: gstreamer::Bin,
}
impl IsElement for Bin {
fn as_element(&self) -> &Element {
let element = self.inner.upcast_ref::<gstreamer::Element>();
unsafe { core::mem::transmute(element) }
}
fn into_element(self) -> Element {
Element {
inner: self.inner.into(),
}
}
}
impl Bin {
pub fn new(name: impl AsRef<str>) -> Self {
let bin = gstreamer::Bin::with_name(name.as_ref());
Bin { inner: bin }
}
pub fn add(&mut self, element: impl IsElement) -> Result<&mut Self> {
self.inner
.add(&element.as_element().inner)
.change_context(Error)
.attach("Failed to add element to bin")?;
Ok(self)
}
pub fn add_many<'a, E: IsElement + 'a>(
&mut self,
elements: impl IntoIterator<Item = &'a E>,
) -> Result<&mut Self> {
self.inner
.add_many(elements.into_iter().map(|e| &e.as_element().inner))
.change_context(Error)
.attach("Failed to add elements to bin")?;
Ok(self)
}
pub fn add_pad(&mut self, pad: &Pad) -> Result<&mut Self> {
self.inner
.add_pad(&pad.inner)
.change_context(Error)
.attach("Failed to add pad to bin")?;
Ok(self)
}
}

53
gst/src/caps.rs Normal file
View File

@@ -0,0 +1,53 @@
use crate::*;
#[repr(transparent)]
pub struct Caps {
pub(crate) inner: gstreamer::caps::Caps,
}
impl Caps {
pub fn builder(cs: CapsType) -> CapsBuilder {
CapsBuilder::new(cs)
}
}
pub struct CapsBuilder {
inner: gstreamer::caps::Builder<gstreamer::caps::NoFeature>,
}
impl CapsBuilder {
pub fn field<V: Into<glib::Value> + Send>(mut self, name: impl AsRef<str>, value: V) -> Self {
use gstreamer::prelude::*;
self.inner = self.inner.field(name.as_ref(), value);
self
}
pub fn build(self) -> Caps {
Caps {
inner: self.inner.build(),
}
}
}
pub enum CapsType {
Video,
Audio,
Text,
}
impl CapsType {
pub fn as_str(&self) -> &str {
match self {
CapsType::Video => "video/x-raw",
CapsType::Audio => "audio/x-raw",
CapsType::Text => "text/x-raw",
}
}
}
impl CapsBuilder {
pub fn new(cs: CapsType) -> Self {
CapsBuilder {
inner: gstreamer::Caps::builder(cs.as_str()),
}
}
}

52
gst/src/element.rs Normal file
View File

@@ -0,0 +1,52 @@
use crate::{Error, Pad, Result, ResultExt};
#[repr(transparent)]
pub struct Element {
pub(crate) inner: gstreamer::Element,
}
pub trait IsElement {
fn as_element(&self) -> &Element;
fn into_element(self) -> Element;
fn pad(&self, name: &str) -> Option<Pad> {
use gstreamer::prelude::*;
self.as_element().inner.static_pad(name).map(Pad::from)
}
}
impl IsElement for Element {
fn as_element(&self) -> &Element {
self
}
fn into_element(self) -> Element {
self
}
}
pub trait Sink: IsElement {
fn sink_pad(&self) -> Pad {
use gstreamer::prelude::*;
self.as_element()
.pad("sink")
.map(From::from)
.expect("Sink element has no sink pad")
}
}
pub trait Source: IsElement {
fn source_pad(&self) -> Pad {
use gstreamer::prelude::*;
self.as_element()
.pad("src")
.map(From::from)
.expect("Source element has no src pad")
}
fn link<S: Sink>(&self, sink: &S) -> Result<()> {
use gstreamer::prelude::*;
self.as_element()
.inner
.link(&sink.as_element().inner)
.change_context(Error)
.attach("Failed to link source to sink")
}
}

View File

@@ -1,5 +1,20 @@
pub mod bin;
pub mod caps;
pub mod element;
pub mod errors;
pub mod playbin3;
pub mod pad;
pub mod plugins;
// pub mod playbin3;
// pub mod videoconvert;
pub use bin::*;
pub use caps::*;
pub use element::*;
pub use pad::*;
pub use plugins::*;
// pub use playbin3::*;
// pub use videoconvert::*;
use errors::*;
use gstreamer::prelude::*;
use std::sync::Arc;
@@ -28,25 +43,18 @@ impl Gst {
Err(_e) => return Err(Error).attach("Failed to downcast to Pipeline"),
Ok(p) => p,
};
Ok(Pipeline { pipeline })
Ok(Pipeline { inner: pipeline })
}
}
pub struct Bin {
bin: gstreamer::Bin,
}
pub struct Sample {
sample: gstreamer::Sample,
}
pub struct Pipeline {
pipeline: gstreamer::Pipeline,
inner: gstreamer::Pipeline,
}
impl core::fmt::Debug for Pipeline {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
f.debug_struct("Pipeline")
.field("pipeline", &self.pipeline)
.field("pipeline", &self.inner)
// .field("state", &self.pipeline.state(gstreamer::ClockTime::NONE))
.finish()
}
@@ -54,23 +62,50 @@ impl core::fmt::Debug for Pipeline {
impl Drop for Pipeline {
fn drop(&mut self) {
let _ = self.pipeline.set_state(gstreamer::State::Null);
let _ = self.inner.set_state(gstreamer::State::Null);
}
}
impl Pipeline {
pub fn bus(&self) -> Result<Bus> {
let bus = self
.pipeline
.inner
.bus()
.ok_or(Error)
.attach("Failed to get bus from pipeline")?;
Ok(Bus { bus })
}
pub fn set_state(&self, state: gstreamer::State) -> Result<gstreamer::StateChangeSuccess> {
pub fn play(&self) -> Result<()> {
self.inner
.set_state(gstreamer::State::Playing)
.change_context(Error)
.attach("Failed to set pipeline to Playing state")?;
Ok(())
}
pub fn pause(&self) -> Result<()> {
self.inner
.set_state(gstreamer::State::Paused)
.change_context(Error)
.attach("Failed to set pipeline to Paused state")?;
Ok(())
}
pub fn ready(&self) -> Result<()> {
self.inner
.set_state(gstreamer::State::Ready)
.change_context(Error)
.attach("Failed to set pipeline to Paused state")?;
Ok(())
}
pub unsafe fn set_state(
&self,
state: gstreamer::State,
) -> Result<gstreamer::StateChangeSuccess> {
let result = self
.pipeline
.inner
.set_state(state)
.change_context(Error)
.attach("Failed to set pipeline state")?;
@@ -93,19 +128,10 @@ impl Bus {
};
self.bus.iter_timed(clocktime)
}
}
/// Pads are link points between elements
pub struct Pad {
pad: gstreamer::Pad,
}
pub struct Element {
element: gstreamer::Element,
}
pub struct AppSink {
appsink: gstreamer_app::AppSink,
pub fn stream(&self) -> gstreamer::bus::BusStream {
self.bus.stream()
}
}
pub struct Playbin3Builder {
@@ -140,7 +166,7 @@ fn gst_play_pipeline() {
let bus = pipeline.bus().expect("Failed to get bus from pipeline");
pipeline
.set_state(gstreamer::State::Playing)
.play()
.expect("Unable to set the pipeline to the `Playing` state");
for msg in bus.iter_timed(None) {
@@ -160,10 +186,6 @@ fn gst_play_pipeline() {
_ => (),
}
}
pipeline
.set_state(gstreamer::State::Null)
.expect("Unable to set the pipeline to the `Null` state");
}
#[test]
@@ -212,13 +234,13 @@ fn test_appsink() {
let bus = pipeline.bus().expect("Failed to get bus from pipeline");
let sink = pipeline
.pipeline
.inner
.by_name("video-sink")
.expect("Sink not found")
.downcast::<gstreamer_app::AppSink>()
.expect("Failed to downcast to AppSink");
let capsfilter = pipeline
.pipeline
.inner
.by_name("video-filter")
.expect("Capsfilter not found");
@@ -236,12 +258,8 @@ fn test_appsink() {
.build(),
);
// let appsink = sink
// .dynamic_cast::<gstreamer_app::AppSink>()
// .expect("Failed to cast to AppSink");
pipeline
.set_state(gstreamer::State::Playing)
.play()
.expect("Unable to set the pipeline to the `Playing` state");
for msg in bus.iter_timed(None) {

31
gst/src/pad.rs Normal file
View File

@@ -0,0 +1,31 @@
use crate::*;
/// Pads are link points between elements
#[repr(transparent)]
pub struct Pad {
pub(crate) inner: gstreamer::Pad,
}
impl From<gstreamer::Pad> for Pad {
fn from(inner: gstreamer::Pad) -> Self {
Pad { inner }
}
}
impl Pad {
pub fn ghost(target: &Pad) -> Result<Pad> {
let ghost_pad = gstreamer::GhostPad::with_target(&target.inner)
.change_context(Error)
.attach("Failed to create ghost pad")?;
Ok(Pad {
inner: ghost_pad.upcast(),
})
}
pub fn activate(&self, activate: bool) -> Result<()> {
use gstreamer::prelude::*;
self.inner
.set_active(activate)
.change_context(Error)
.attach("Failed to set_active pad")?;
Ok(())
}
}

View File

@@ -1,65 +0,0 @@
use crate::*;
pub struct Playbin3 {
inner: gstreamer::Element,
}
impl Drop for Playbin3 {
fn drop(&mut self) {
let _ = self.inner.set_state(gstreamer::State::Null);
}
}
impl Playbin3 {
pub fn new(name: impl AsRef<str>) -> Result<Self> {
use gstreamer::prelude::*;
gstreamer::ElementFactory::make("playbin3")
.name(name.as_ref())
.build()
.map(|element| Playbin3 { inner: element })
.change_context(Error)
}
pub fn with_uri(self, uri: impl AsRef<str>) -> Self {
use gstreamer::prelude::*;
self.inner.set_property("uri", uri.as_ref());
self
}
pub fn sample(&self) -> Option<Sample> {
use gstreamer::prelude::*;
self.inner
.property::<Option<gstreamer::Sample>>("sample")
// // .and_then(|v| v.get::<gstreamer::Sample>().ok())
.map(|sample| Sample { sample })
}
pub fn play(&self) -> Result<()> {
use gstreamer::prelude::*;
self.inner
.set_state(gstreamer::State::Playing)
.change_context(Error)
.attach("Failed to set playbin3 to Playing state")?;
Ok(())
}
pub fn bus(&self) -> Result<Bus> {
let bus = self
.inner
.bus()
.ok_or(Error)
.attach("Failed to get bus from playbin3")?;
Ok(Bus { bus })
}
}
#[test]
fn test_playbin3() {
use gstreamer::prelude::*;
gstreamer::init().unwrap();
let playbin3 = Playbin3::new("test_playbin3").unwrap().with_uri(
"https://www.freedesktop.org/software/gstreamer-sdk/data/media/sintel_trailer-480p.webm",
);
playbin3.play().unwrap();
let bus = playbin3.bus().unwrap();
for msg in bus.iter_timed(None) {}
// std::thread::sleep(std::time::Duration::from_secs(5));
}

4
gst/src/plugins.rs Normal file
View File

@@ -0,0 +1,4 @@
pub mod app;
pub mod autodetect;
pub mod playback;
pub mod videoconvertscale;

2
gst/src/plugins/app.rs Normal file
View File

@@ -0,0 +1,2 @@
pub mod appsink;
pub use appsink::*;

View File

@@ -0,0 +1,108 @@
use crate::*;
pub struct AppSink {
inner: gstreamer::Element,
}
impl IsElement for AppSink {
fn as_element(&self) -> &Element {
unsafe { core::mem::transmute(&self.inner) }
}
fn into_element(self) -> Element {
Element { inner: self.inner }
}
}
impl Sink for AppSink {}
impl AppSink {
fn appsink(&self) -> &gstreamer_app::AppSink {
self.inner
.downcast_ref::<gstreamer_app::AppSink>()
.expect("Failed to downcast to AppSink")
}
pub fn new(name: impl AsRef<str>) -> Result<Self> {
use gstreamer::prelude::*;
let inner = gstreamer::ElementFactory::make("appsink")
.name(name.as_ref())
.build()
.change_context(Error)
.attach("Failed to create appsink element")?;
Ok(AppSink { inner })
}
pub fn with_caps(mut self, caps: &gstreamer::Caps) -> Self {
use gstreamer::prelude::*;
// self.inner.set_caps(Some(caps));
self
}
pub fn set_callbacks(&self, callbacks: gstreamer_app::AppSinkCallbacks) -> Result<()> {
self.appsink().set_callbacks(callbacks);
Ok(())
}
pub fn pull_sample(&self, timeout: impl Into<Option<core::time::Duration>>) -> Result<Sample> {
use gstreamer::prelude::*;
self.appsink()
.pull_sample()
.change_context(Error)
.attach("Failed to pull sample from AppSink")
.map(Sample::from)
}
pub fn try_pull_sample(
&self,
timeout: impl Into<Option<core::time::Duration>>,
) -> Result<Option<Sample>> {
use gstreamer::prelude::*;
Ok(self
.appsink()
.try_pull_sample(duration_to_clocktime(timeout)?)
.map(From::from))
}
pub fn pull_preroll(&self, timeout: impl Into<Option<core::time::Duration>>) -> Result<Sample> {
use gstreamer::prelude::*;
self.appsink()
.pull_preroll()
.change_context(Error)
.attach("Failed to pull preroll sample from AppSink")
.map(Sample::from)
}
pub fn try_pull_preroll(
&self,
timeout: impl Into<Option<core::time::Duration>>,
) -> Result<Option<Sample>> {
use gstreamer::prelude::*;
Ok(self
.appsink()
.try_pull_preroll(duration_to_clocktime(timeout)?)
.map(From::from))
}
}
fn duration_to_clocktime(
timeout: impl Into<Option<core::time::Duration>>,
) -> Result<Option<gstreamer::ClockTime>> {
match timeout.into() {
Some(dur) => {
let clocktime = gstreamer::ClockTime::try_from(dur)
.change_context(Error)
.attach("Failed to convert duration to ClockTime")?;
Ok(Some(clocktime))
}
None => Ok(None),
}
}
pub struct Sample {
inner: gstreamer::Sample,
}
impl From<gstreamer::Sample> for Sample {
fn from(inner: gstreamer::Sample) -> Self {
Sample { inner }
}
}

View File

@@ -0,0 +1,2 @@
pub mod autovideosink;
pub use autovideosink::*;

View File

@@ -0,0 +1,30 @@
use crate::*;
#[repr(transparent)]
pub struct AutoVideoSink {
inner: gstreamer::Element,
}
impl IsElement for AutoVideoSink {
fn as_element(&self) -> &Element {
unsafe { core::mem::transmute(&self.inner) }
}
fn into_element(self) -> Element {
Element { inner: self.inner }
}
}
impl Sink for AutoVideoSink {}
impl AutoVideoSink {
pub fn new(name: impl AsRef<str>) -> Result<Self> {
use gstreamer::prelude::*;
let element = gstreamer::ElementFactory::make("autovideosink")
.name(name.as_ref())
.build()
.change_context(Error)
.attach("Failed to create autovideosink element")?;
Ok(AutoVideoSink { inner: element })
}
}

View File

@@ -0,0 +1,2 @@
pub mod playbin3;
pub use playbin3::*;

View File

@@ -0,0 +1,125 @@
use crate::*;
pub struct Playbin3 {
inner: gstreamer::Element,
}
impl Drop for Playbin3 {
fn drop(&mut self) {
let _ = self
.inner
.set_state(gstreamer::State::Null)
.inspect_err(|e| {
tracing::error!("Failed to set playbin3 to Null state on drop: {:?}", e)
});
}
}
impl Playbin3 {
pub fn new(name: impl AsRef<str>) -> Result<Self> {
use gstreamer::prelude::*;
gstreamer::ElementFactory::make("playbin3")
.name(name.as_ref())
.build()
.map(|element| Playbin3 { inner: element })
.change_context(Error)
}
pub fn with_uri(self, uri: impl AsRef<str>) -> Self {
use gstreamer::prelude::*;
self.inner.set_property("uri", uri.as_ref());
self
}
pub fn with_video_sink(self, video_sink: &impl IsElement) -> Self {
use gstreamer::prelude::*;
self.inner
.set_property("video-sink", &video_sink.as_element().inner);
self
}
pub fn with_text_sink(self, text_sink: &impl IsElement) -> Self {
use gstreamer::prelude::*;
self.inner
.set_property("text-sink", &text_sink.as_element().inner);
self
}
pub fn with_audio_sink(self, audio_sink: &impl IsElement) -> Self {
use gstreamer::prelude::*;
self.inner
.set_property("audio-sink", &audio_sink.as_element().inner);
self
}
pub fn set_volume(&self, volume: f64) {
use gstreamer::prelude::*;
self.inner.set_property("volume", volume.clamp(1.0, 100.0))
}
pub fn get_volume(&self) -> f64 {
use gstreamer::prelude::*;
self.inner.property::<f64>("volume")
}
pub fn play(&self) -> Result<()> {
use gstreamer::prelude::*;
self.inner
.set_state(gstreamer::State::Playing)
.change_context(Error)
.attach("Failed to set playbin3 to Playing state")?;
Ok(())
}
pub fn bus(&self) -> Result<Bus> {
let bus = self
.inner
.bus()
.ok_or(Error)
.attach("Failed to get bus from playbin3")?;
Ok(Bus { bus })
}
}
#[test]
fn test_playbin3() {
use gstreamer::prelude::*;
use tracing_subscriber::prelude::*;
tracing_subscriber::registry()
.with(
tracing_subscriber::fmt::layer()
.with_thread_ids(true)
.with_file(true),
)
.init();
tracing::info!("Linking videoconvert to appsink");
gstreamer::init().unwrap();
let playbin3 = Playbin3::new("test_playbin3").unwrap().with_uri("https://jellyfin.tsuba.darksailor.dev/Items/6010382cf25273e624d305907010d773/Download?api_key=036c140222464878862231ef66a2bc9c");
// let mut video_sink = Bin::new("wgpu_video_sink");
//
// let video_convert = plugins::videoconvertscale::VideoConvert::new("wgpu_video_convert")
// .expect("Create videoconvert");
// let appsink = AppSink::new("test_appsink").expect("Create appsink");
let appsink = plugins::autodetect::AutoVideoSink::new("test_autodetect_video_sink")
.expect("Create autodetect video sink");
// video_convert
// .link(&appsink)
// .expect("Link videoconvert to appsink");
//
// let sink_pad = video_convert.sink_pad();
// let sink_pad = Pad::ghost(&sink_pad).expect("Create ghost pad from videoconvert src");
// video_sink
// .add(appsink)
// .expect("Add appsink to video sink")
// .add(video_convert)
// .expect("Add videoconvert to video sink")
// .add_pad(&sink_pad)
// .expect("Add ghost pad to video sink");
// sink_pad.activate(true).expect("Activate ghost pad");
let playbin3 = playbin3.with_video_sink(&appsink);
playbin3.play().unwrap();
let bus = playbin3.bus().unwrap();
for msg in bus.iter_timed(None) {
tracing::info!("{:#?}", &msg.view());
}
// std::thread::sleep(std::time::Duration::from_secs(5));
}

View File

@@ -0,0 +1,2 @@
pub mod videoconvert;
pub use videoconvert::*;

View File

@@ -0,0 +1,49 @@
use crate::*;
#[doc(inline)]
pub use gstreamer_video::VideoFormat;
#[repr(transparent)]
pub struct VideoConvert {
inner: gstreamer::Element,
}
impl IsElement for VideoConvert {
fn as_element(&self) -> &Element {
unsafe { core::mem::transmute(&self.inner) }
}
fn into_element(self) -> Element {
Element { inner: self.inner }
}
}
impl Sink for VideoConvert {}
impl Source for VideoConvert {}
impl VideoConvert {
pub fn new(name: impl AsRef<str>) -> Result<Self> {
use gstreamer::prelude::*;
let element = gstreamer::ElementFactory::make("videoconvert")
.name(name.as_ref())
.build()
.change_context(Error)
.attach("Failed to create videoconvert element")?;
Ok(VideoConvert { inner: element })
}
// pub fn with_caps(mut self, caps: &gstreamer::Caps) -> Self {
// use gstreamer::prelude::*;
// self.inner.set_property("caps", caps);
// self
// }
pub fn with_output_format(self, format: VideoFormat) -> Result<Self> {
use gstreamer::prelude::*;
let caps = Caps::builder(CapsType::Video)
.field("format", format.to_str())
.build();
self.inner.set_property("caps", &caps.inner);
// .change_context(Error)
// .attach("Failed to set output format on videoconvert")?;
Ok(self)
}
}

View File

@@ -0,0 +1,3 @@
// pub fn copy_sample_to_texture() {
//
// }