servius/jello
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Modify gst crate to add lot of more granularity

Browse Source
This commit is contained in:
uttarayan21
2025-12-22 13:27:30 +05:30
parent d42ef3b550
commit 043d1e99f0
23 changed files with 947 additions and 392 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
Cargo.lock generated
View File

@@ -3050,6 +3050,7 @@ name = "gst"
version = "0.1.0"
dependencies = [
"error-stack",
"futures",
"glib 0.21.5",
"gstreamer 0.24.4",
"gstreamer-app 0.24.4",
@@ -3607,6 +3608,18 @@ dependencies = [
"thiserror 2.0.17",
]
[[package]]
name = "iced-video"
version = "0.1.0"
dependencies = [
"error-stack",
"gst",
"iced_core",
"iced_wgpu",
"thiserror 2.0.17",
"tracing",
]
[[package]]
name = "iced_beacon"
version = "0.14.0"

10
Cargo.toml
View File

@@ -9,16 +9,10 @@ members = [
"jello-types",
"gst",
"examples/hdr-gstreamer-wgpu",
"crates/iced-video",
]
[workspace.dependencies]
iced = { version = "0.14.0", features = [
"advanced",
"canvas",
"image",
"sipper",
"tokio",
"debug",
] }
iced = { version = "0.14.0" }
iced_video_player = "0.6"
gst = { version = "0.1.0", path = "gst" }
# iced_video_player = { git = "https://github.com/jazzfool/iced_video_player" }

12
crates/iced-video/Cargo.toml Normal file
View File

@@ -0,0 +1,12 @@
[package]
name = "iced-video"
version = "0.1.0"
edition = "2024"
[dependencies]
error-stack = "0.6.0"
gst.workspace = true
iced_core = "0.14.0"
iced_wgpu = "0.14.0"
thiserror = "2.0.17"
tracing = "0.1.43"

123
crates/iced-video/src/lib.rs Normal file
View File

@@ -0,0 +1,123 @@
pub mod primitive;
pub mod source;
use error_stack::{Report, ResultExt};
use gst::*;
use iced_core::Length;
use std::marker::PhantomData;
use gst::plugins::app::AppSink;
use gst::plugins::playback::Playbin3;
use gst::plugins::videoconvertscale::VideoConvert;
#[derive(Debug, thiserror::Error)]
#[error("Iced Video Error")]
pub struct Error;
pub type Result<T, E = Report<Error>> = core::result::Result<T, E>;
use std::sync::{Arc, Mutex, atomic::AtomicBool};
pub struct Video {
id: iced_core::Id,
source: source::VideoSource,
is_playing: Arc<AtomicBool>,
is_eos: Arc<AtomicBool>,
texture: Mutex<Option<iced_wgpu::wgpu::TextureView>>,
}
impl Video {
pub fn id(&self) -> &iced_core::Id {
&self.id
}
pub fn source(&self) -> &source::VideoSource {
&self.source
}
pub async fn new(url: impl AsRef<str>) -> Result<Self> {
Ok(Self {
id: iced_core::Id::unique(),
source: source::VideoSource::new(url)?,
is_playing: Arc::new(AtomicBool::new(false)),
is_eos: Arc::new(AtomicBool::new(false)),
texture: Mutex::new(None),
})
}
}
pub struct VideoPlayer<'a, Message, Theme = iced_core::Theme, Renderer = iced_wgpu::Renderer>
where
Renderer: PrimitiveRenderer,
{
videos: &'a Video,
content_fit: iced_core::ContentFit,
width: iced_core::Length,
height: iced_core::Length,
on_end_of_stream: Option<Message>,
on_new_frame: Option<Message>,
looping: bool,
// on_subtitle_text: Option<Box<dyn Fn(Option<String>) -> Message + 'a>>,
// on_error: Option<Box<dyn Fn(&glib::Error) -> Message + 'a>>,
theme: Theme,
__marker: PhantomData<Renderer>,
}
impl<Message, Theme, Renderer> VideoPlayer<Message, Theme, Renderer>
where
Renderer: PrimitiveRenderer,
{
pub fn new(source: source::VideoSource) -> Self {
Self {
videos: Video {
id: iced_core::Id::unique(),
source,
is_playing: Arc::new(AtomicBool::new(false)),
is_eos: Arc::new(AtomicBool::new(false)),
texture: Mutex::new(None),
},
content_fit: iced_core::ContentFit::Contain,
width: Length::Shrink,
height: Length::Shrink,
on_end_of_stream: None,
on_new_frame: None,
looping: false,
theme: Theme::default(),
__marker: PhantomData,
}
}
}
impl<Message, Theme, Renderer> iced_core::Widget<Message, Theme, Renderer>
for VideoPlayer<'_, Message, Theme, Renderer>
where
Message: Clone,
Renderer: PrimitiveRenderer,
{
fn size(&self) -> iced_core::Size<Length> {
iced_core::Size {
width: self.width,
height: self.height,
}
}
fn layout(
&mut self,
iced_core::widget::tree: &mut iced_core::widget::Tree,
iced_core::renderer: &Renderer,
limits: &iced_core::layout::Limits,
) -> iced_core::layout::Node {
todo!()
}
fn draw(
&self,
iced_core::widget::tree: &iced_core::widget::Tree,
iced_core::renderer: &mut Renderer,
theme: &Theme,
style: &iced_core::renderer::Style,
iced_core::layout: iced_core::Layout<'_>,
cursor: iced_core::mouse::Cursor,
viewport: &iced_core::Rectangle,
) {
todo!()
}
}

177
crates/iced-video/src/primitive.rs Normal file
View File

@@ -0,0 +1,177 @@
use iced_wgpu::primitive::Pipeline;
use iced_wgpu::wgpu;
use std::collections::BTreeMap;
use std::sync::{Arc, atomic::AtomicBool};
#[derive(Debug)]
pub struct VideoPrimitive {
texture: wgpu::TextureView,
ready: Arc<AtomicBool>,
}
impl iced_wgpu::Primitive for VideoPrimitive {
type Pipeline = VideoPipeline;
fn prepare(
&self,
pipeline: &mut Self::Pipeline,
device: &wgpu::Device,
queue: &wgpu::Queue,
bounds: &iced_wgpu::core::Rectangle,
viewport: &iced_wgpu::graphics::Viewport,
) {
todo!()
}
fn draw(&self, _pipeline: &Self::Pipeline, _render_pass: &mut wgpu::RenderPass<'_>) -> bool {
false
}
fn render(
&self,
pipeline: &Self::Pipeline,
encoder: &mut wgpu::CommandEncoder,
target: &wgpu::TextureView,
clip_bounds: &iced_wgpu::core::Rectangle<u32>,
) {
if self.ready.load(std::sync::atomic::Ordering::SeqCst) {
let mut render_pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor {
label: Some("iced-video-render-pass"),
color_attachments: &[Some(wgpu::RenderPassColorAttachment {
view: target,
resolve_target: None,
ops: wgpu::Operations {
load: wgpu::LoadOp::Load,
store: wgpu::StoreOp::Store,
},
depth_slice: None,
})],
depth_stencil_attachment: None,
timestamp_writes: None,
occlusion_query_set: None,
});
render_pass.set_pipeline(&pipeline.pipeline);
render_pass.set_bind_group(0, &self.bind_group, &[]);
render_pass.draw(0..3, 0..1);
self.ready
.store(false, std::sync::atomic::Ordering::Relaxed);
}
}
}
#[derive(Debug)]
pub struct VideoTextures {
id: u64,
texture: wgpu::Texture,
bind_group: wgpu::BindGroup,
ready: Arc<AtomicBool>,
}
#[derive(Debug)]
pub struct VideoPipeline {
pipeline: wgpu::RenderPipeline,
bind_group_layout: wgpu::BindGroupLayout,
sampler: wgpu::Sampler,
videos: BTreeMap<u64, VideoTextures>,
}
pub trait HdrTextureFormatExt {
fn is_hdr(&self) -> bool;
}
impl HdrTextureFormatExt for wgpu::TextureFormat {
fn is_hdr(&self) -> bool {
matches!(
self,
wgpu::TextureFormat::Rgba16Float
| wgpu::TextureFormat::Rgba32Float
| wgpu::TextureFormat::Rgb10a2Unorm
| wgpu::TextureFormat::Rgb10a2Uint
)
}
}
impl Pipeline for VideoPipeline {
fn new(device: &wgpu::Device, queue: &wgpu::Queue, format: wgpu::TextureFormat) -> Self
where
Self: Sized,
{
if format.is_hdr() {
tracing::info!("HDR texture format detected: {:?}", format);
}
let shader_passthrough =
device.create_shader_module(wgpu::include_wgsl!("shaders/passthrough.wgsl"));
let bind_group_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor {
label: Some("iced-video-texture-bind-group-layout"),
entries: &[
wgpu::BindGroupLayoutEntry {
binding: 0,
visibility: wgpu::ShaderStages::FRAGMENT,
ty: wgpu::BindingType::Texture {
multisampled: false,
view_dimension: wgpu::TextureViewDimension::D2,
sample_type: wgpu::TextureSampleType::Float { filterable: true },
},
count: None,
},
wgpu::BindGroupLayoutEntry {
binding: 1,
visibility: wgpu::ShaderStages::FRAGMENT,
ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering),
count: None,
},
],
});
let render_pipeline_layout =
device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
label: Some("iced-video-render-pipeline-layout"),
bind_group_layouts: &[&bind_group_layout],
push_constant_ranges: &[],
});
let pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor {
label: Some("iced-video-render-pipeline"),
layout: Some(&render_pipeline_layout),
vertex: wgpu::VertexState {
module: &shader_passthrough,
entry_point: Some("vs_main"),
buffers: &[],
compilation_options: wgpu::PipelineCompilationOptions::default(),
},
fragment: Some(wgpu::FragmentState {
module: &shader_passthrough,
entry_point: Some("fs_main"),
targets: &[Some(wgpu::ColorTargetState {
format,
blend: Some(wgpu::BlendState::ALPHA_BLENDING),
write_mask: wgpu::ColorWrites::ALL,
})],
compilation_options: wgpu::PipelineCompilationOptions::default(),
}),
primitive: wgpu::PrimitiveState::default(),
depth_stencil: None,
multisample: wgpu::MultisampleState::default(),
multiview: None,
cache: None,
});
let sampler = device.create_sampler(&wgpu::SamplerDescriptor {
label: Some("iced-video-sampler"),
address_mode_u: wgpu::AddressMode::ClampToEdge,
address_mode_v: wgpu::AddressMode::ClampToEdge,
address_mode_w: wgpu::AddressMode::ClampToEdge,
mag_filter: wgpu::FilterMode::Linear,
min_filter: wgpu::FilterMode::Linear,
mipmap_filter: wgpu::FilterMode::Nearest,
..Default::default()
});
Self {
pipeline,
bind_group_layout,
sampler,
videos: BTreeMap::new(),
}
}
}

0
crates/iced-video/src/shaders/bt.2020.wgsl Normal file
View File

0
crates/iced-video/src/shaders/bt.709.wgsl Normal file
View File

31
crates/iced-video/src/shaders/passthrough.wgsl Normal file
View File

@@ -0,0 +1,31 @@
// Vertex shader
struct VertexOutput {
@builtin(position) clip_position: vec4<f32>,
@location(0) tex_coords: vec2<f32>,
};
@vertex
fn vs_main(
@builtin(vertex_index) in_vertex_index: u32,
) -> VertexOutput {
var out: VertexOutput;
let uv = vec2<f32>(f32((in_vertex_index << 1u) & 2u), f32(in_vertex_index & 2u));
out.clip_position = vec4<f32>(uv * 2.0 - 1.0, 0.0, 1.0);
out.clip_position.y = -out.clip_position.y;
out.tex_coords = uv;
return out;
}
// Fragment shader
@group(0) @binding(0)
var t_diffuse: texture_2d<f32>;
@group(0) @binding(1)
var s_diffuse: sampler;
@fragment
fn fs_main(in: VertexOutput) -> @location(0) vec4<f32> {
return textureSample(t_diffuse, s_diffuse, in.tex_coords);
}

64
crates/iced-video/src/source.rs Normal file
View File

@@ -0,0 +1,64 @@
#[derive(Debug, Clone)]
pub struct VideoSource {
playbin: Playbin3,
videoconvert: VideoConvert,
appsink: AppSink,
bus: Bus,
}
impl VideoSource {
/// Creates a new video source from the given URL.
/// Since this doesn't have to parse the pipeline manually, we aren't sanitizing the URL for
/// now.
pub async fn new(url: impl AsRef<str>) -> Result<Self> {
Gst::new();
let videoconvert = VideoConvert::new("iced-video-convert").change_context(Error)?;
let appsink = AppSink::new("iced-video-sink").change_context(Error)?;
let video_sink = videoconvert.link(&appsink).change_context(Error)?;
let playbin = gst::plugins::playback::Playbin3::new("iced-video")
.change_context(Error)?
.with_uri(url.as_ref())
.with_video_sink(&video_sink);
let bus = playbin.bus().change_context(Error)?;
playbin.wait_ready()?;
// let bus_stream = bus.stream();
// bus_stream.find(|message| {
// let view = message.view();
// if let gst::MessageView::StateChanged(change) = view {
// change.current() == gst::State::Ready
// } else {
// false
// }
// });
Ok(Self {
playbin,
videoconvert,
appsink,
bus,
})
}
pub fn play(&self) -> Result<()> {
self.playbin
.play()
.change_context(Error)
.attach("Failed to play video")
}
pub fn pause(&self) -> Result<()> {
self.playbin
.pause()
.change_context(Error)
.attach("Failed to pause video")
}
pub fn bus(&self) -> &Bus {}
// pub fn copy_frame_to_texture(&self, texture: wgpu::TextureView) -> Result<()> {
// let frame = self
// .appsink
// .try_pull_sample(core::time::Duration::from_millis(1))?
// .ok_or(Error)
// .attach("No video frame available")?;
// }
}

3
gst/Cargo.toml
View File

@@ -1,12 +1,13 @@
[package]
name = "gst"
version = "0.1.0"
edition = "2021"
edition = "2024"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
error-stack = "0.6"
futures = "0.3.31"
glib = "0.21.5"
gstreamer = { version = "0.24.4", features = ["v1_18"] }
gstreamer-app = { version = "0.24.4", features = ["v1_18"] }

9
gst/src/bin.rs
View File

@@ -1,9 +1,16 @@
use crate::*;
#[repr(transparent)]
pub struct Bin {
inner: gstreamer::Bin,
}
impl From<gstreamer::Bin> for Bin {
fn from(inner: gstreamer::Bin) -> Self {
Bin { inner }
}
}
impl IsElement for Bin {
fn as_element(&self) -> &Element {
let element = self.inner.upcast_ref::<gstreamer::Element>();
@@ -22,7 +29,7 @@ impl Bin {
Bin { inner: bin }
}
pub fn add(&mut self, element: impl IsElement) -> Result<&mut Self> {
pub fn add(&mut self, element: &impl IsElement) -> Result<&mut Self> {
self.inner
.add(&element.as_element().inner)
.change_context(Error)

26
gst/src/bus.rs Normal file
View File

@@ -0,0 +1,26 @@
#[derive(Debug, Clone)]
#[repr(transparent)]
pub struct Bus {
pub(crate) bus: gstreamer::Bus,
}
impl Bus {
pub fn iter_timed(
&self,
timeout: impl Into<Option<core::time::Duration>>,
) -> gstreamer::bus::Iter<'_> {
let clocktime = match timeout.into() {
Some(dur) => gstreamer::ClockTime::try_from(dur).ok(),
None => gstreamer::ClockTime::NONE,
};
self.bus.iter_timed(clocktime)
}
pub fn stream(&self) -> gstreamer::bus::BusStream {
self.bus.stream()
}
pub fn into_inner(self) -> gstreamer::Bus {
self.bus
}
}

9
gst/src/caps.rs
View File

@@ -51,3 +51,12 @@ impl CapsBuilder {
}
}
}
impl Caps {
pub fn format(&self) -> Option<&str> {
use gstreamer::prelude::*;
self.inner
.structure(0)
.and_then(|s| s.get::<&str>("format").ok())
}
}

78
gst/src/element.rs
View File

@@ -1,4 +1,4 @@
use crate::{Error, Pad, Result, ResultExt};
use crate::{Bin, Error, Pad, Result, ResultExt};
#[repr(transparent)]
pub struct Element {
pub(crate) inner: gstreamer::Element,
@@ -24,29 +24,85 @@ impl IsElement for Element {
}
pub trait Sink: IsElement {
fn sink_pad(&self) -> Pad {
fn sink(&self, name: impl AsRef<str>) -> Pad {
use gstreamer::prelude::*;
self.as_element()
.pad("sink")
.pad(name.as_ref())
.map(From::from)
.expect("Sink element has no sink pad")
}
}
pub trait Source: IsElement {
fn source_pad(&self) -> Pad {
fn source(&self, name: impl AsRef<str>) -> Pad {
use gstreamer::prelude::*;
self.as_element()
.pad("src")
.pad(name.as_ref())
.map(From::from)
.expect("Source element has no src pad")
}
fn link<S: Sink>(&self, sink: &S) -> Result<()> {
fn link<S: Sink>(&self, sink: &S) -> Result<Bin>
where
Self: Sized,
{
use gstreamer::prelude::*;
self.as_element()
.inner
.link(&sink.as_element().inner)
.change_context(Error)
.attach("Failed to link source to sink")
if let Ok(bin) = self.as_element().inner.clone().downcast::<gstreamer::Bin>() {
bin.add(&sink.as_element().inner)
.change_context(Error)
.attach("Failed to add sink to bin")?;
self.as_element()
.inner
.link(&sink.as_element().inner)
.change_context(Error)
.attach("Failed to link elements")?;
Ok(Bin::from(bin))
} else {
let bin = gstreamer::Bin::builder()
.name(format!(
"{}-link-{}",
self.as_element().inner.name(),
sink.as_element().inner.name()
))
.build();
bin.add(&self.as_element().inner)
.change_context(Error)
.attach("Failed to add source to bin")?;
bin.add(&sink.as_element().inner)
.change_context(Error)
.attach("Failed to add sink to bin")?;
self.as_element()
.inner
.link(&sink.as_element().inner)
.change_context(Error)
.attach("Failed to link elements")?;
if let Some(sink_pad) = self.as_element().pad("sink") {
let ghost_pad = Pad::ghost(&sink_pad)?;
bin.add_pad(&ghost_pad.inner)
.change_context(Error)
.attach("Failed to add src pad to bin")?;
ghost_pad.activate(true)?;
}
Ok(From::from(bin))
}
}
// fn link_pad<S: Sink>(&self, sink: &S, src_pad_name: &str, sink_pad_name: &str) -> Result<()> {
// use gstreamer::prelude::*;
// let src_pad = self
// .as_element()
// .pad(src_pad_name)
// .ok_or(Error)
// .attach("Source pad not found")?;
// let sink_pad = sink
// .as_element()
// .pad(sink_pad_name)
// .ok_or(Error)
// .attach("Sink pad not found")?;
// src_pad
// .inner
// .link(&sink_pad.inner)
// .change_context(Error)
// .attach("Failed to link source pad to sink pad")?;
// Ok(())
// }
}

19
gst/src/isa.rs Normal file
View File

@@ -0,0 +1,19 @@
// use crate::errors::*;
// /// This trait is used for implementing parent traits methods on children.
// pub trait Upcastable<T> {
// #[track_caller]
// fn upcast(self) -> T;
// fn upcast_ref(&self) -> &T;
// }
//
// // impl Upcastable<GenericPipeline> for crate::playback::Playbin3 {}
// impl<P, C> core::ops::Deref for C
// where
// C: Upcastable<P>,
// {
// type Target = P;
//
// fn deref(&self) -> &Self::Target {
// todo!()
// }
// }

316
gst/src/lib.rs
View File

@@ -1,19 +1,40 @@
pub mod bin;
pub mod bus;
pub mod caps;
pub mod element;
pub mod errors;
pub mod isa;
pub mod pad;
pub mod pipeline;
pub mod plugins;
// pub mod playbin3;
// pub mod videoconvert;
pub use bin::*;
pub use bus::*;
pub use caps::*;
pub use element::*;
pub use pad::*;
pub use pipeline::*;
pub use plugins::*;
// pub use playbin3::*;
// pub use videoconvert::*;
pub(crate) mod priv_prelude {
pub use crate::errors::*;
pub use crate::*;
pub use gstreamer::prelude::*;
#[track_caller]
pub fn duration_to_clocktime(
timeout: impl Into<Option<core::time::Duration>>,
) -> Result<Option<gstreamer::ClockTime>> {
match timeout.into() {
Some(dur) => {
let clocktime = gstreamer::ClockTime::try_from(dur)
.change_context(Error)
.attach("Failed to convert duration to ClockTime")?;
Ok(Some(clocktime))
}
None => Ok(None),
}
}
}
use errors::*;
use gstreamer::prelude::*;
@@ -36,282 +57,13 @@ impl Gst {
Arc::clone(&GST)
}
pub fn pipeline_from_str(&self, s: &str) -> Result<Pipeline> {
let pipeline = gstreamer::parse::launch(s).change_context(Error)?;
let pipeline = pipeline.downcast::<gstreamer::Pipeline>();
let pipeline = match pipeline {
Err(_e) => return Err(Error).attach("Failed to downcast to Pipeline"),
Ok(p) => p,
};
Ok(Pipeline { inner: pipeline })
}
}
pub struct Pipeline {
inner: gstreamer::Pipeline,
}
impl core::fmt::Debug for Pipeline {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
f.debug_struct("Pipeline")
.field("pipeline", &self.inner)
// .field("state", &self.pipeline.state(gstreamer::ClockTime::NONE))
.finish()
}
}
impl Drop for Pipeline {
fn drop(&mut self) {
let _ = self.inner.set_state(gstreamer::State::Null);
}
}
impl Pipeline {
pub fn bus(&self) -> Result<Bus> {
let bus = self
.inner
.bus()
.ok_or(Error)
.attach("Failed to get bus from pipeline")?;
Ok(Bus { bus })
}
pub fn play(&self) -> Result<()> {
self.inner
.set_state(gstreamer::State::Playing)
.change_context(Error)
.attach("Failed to set pipeline to Playing state")?;
Ok(())
}
pub fn pause(&self) -> Result<()> {
self.inner
.set_state(gstreamer::State::Paused)
.change_context(Error)
.attach("Failed to set pipeline to Paused state")?;
Ok(())
}
pub fn ready(&self) -> Result<()> {
self.inner
.set_state(gstreamer::State::Ready)
.change_context(Error)
.attach("Failed to set pipeline to Paused state")?;
Ok(())
}
pub unsafe fn set_state(
&self,
state: gstreamer::State,
) -> Result<gstreamer::StateChangeSuccess> {
let result = self
.inner
.set_state(state)
.change_context(Error)
.attach("Failed to set pipeline state")?;
Ok(result)
}
}
pub struct Bus {
bus: gstreamer::Bus,
}
impl Bus {
pub fn iter_timed(
&self,
timeout: impl Into<Option<core::time::Duration>>,
) -> gstreamer::bus::Iter<'_> {
let clocktime = match timeout.into() {
Some(dur) => gstreamer::ClockTime::try_from(dur).ok(),
None => gstreamer::ClockTime::NONE,
};
self.bus.iter_timed(clocktime)
}
pub fn stream(&self) -> gstreamer::bus::BusStream {
self.bus.stream()
}
}
pub struct Playbin3Builder {
uri: Option<String>,
video_sink: Option<Element>,
audio_sink: Option<Element>,
text_sink: Option<Element>,
}
#[test]
fn gst_parse_pipeline() {
let gst = Gst::new();
let pipeline = gst
.pipeline_from_str("videotestsrc ! autovideosink")
.expect("Failed to create pipeline");
println!("{:?}", pipeline);
}
#[test]
fn gst_parse_invalid_pipeline() {
let gst = Gst::new();
let result = gst.pipeline_from_str("invalidpipeline");
assert!(result.is_err());
}
#[test]
fn gst_play_pipeline() {
let gst = Gst::new();
let pipeline = gst
.pipeline_from_str("videotestsrc ! autovideosink")
.expect("Failed to create pipeline");
let bus = pipeline.bus().expect("Failed to get bus from pipeline");
pipeline
.play()
.expect("Unable to set the pipeline to the `Playing` state");
for msg in bus.iter_timed(None) {
use gstreamer::MessageView;
match msg.view() {
MessageView::Eos(..) => break,
MessageView::Error(err) => {
eprintln!(
"Error from {:?}: {} ({:?})",
err.src().map(|s| s.path_string()),
err.error(),
err.debug()
);
break;
}
_ => (),
}
}
}
#[test]
#[ignore]
fn gstreamer_unwrapped() {
gstreamer::init();
let uri = "https://gstreamer.freedesktop.org/data/media/sintel_trailer-480p.webm";
let pipeline = gstreamer::parse::launch(&format!("playbin uri={}", uri)).unwrap();
use gstreamer::prelude::*;
pipeline.set_state(gstreamer::State::Playing).unwrap();
let bus = pipeline.bus().unwrap();
for msg in bus.iter_timed(gstreamer::ClockTime::NONE) {
use gstreamer::MessageView;
match msg.view() {
MessageView::Eos(..) => break,
MessageView::Error(err) => {
eprintln!(
"Error from {:?}: {} ({:?})",
err.src().map(|s| s.path_string()),
err.error(),
err.debug()
);
break;
}
_ => (),
}
}
pipeline.set_state(gstreamer::State::Null).unwrap();
}
#[test]
fn test_appsink() {
let gst = Gst::new();
let pipeline = gst
.pipeline_from_str(
"videotestsrc ! videoconvert | capsfilter name=video-filter ! appsink name=video-sink",
)
.expect("Failed to create pipeline");
// let video_sink = pipeline.
let bus = pipeline.bus().expect("Failed to get bus from pipeline");
let sink = pipeline
.inner
.by_name("video-sink")
.expect("Sink not found")
.downcast::<gstreamer_app::AppSink>()
.expect("Failed to downcast to AppSink");
let capsfilter = pipeline
.inner
.by_name("video-filter")
.expect("Capsfilter not found");
let caps = gstreamer::Caps::builder("video/x-raw")
.field("format", "RGBA")
.build();
capsfilter.set_property("caps", &caps);
sink.set_callbacks(
gstreamer_app::AppSinkCallbacks::builder()
.new_sample(|sink| {
// foo
Ok(gstreamer::FlowSuccess::Ok)
})
.build(),
);
pipeline
.play()
.expect("Unable to set the pipeline to the `Playing` state");
for msg in bus.iter_timed(None) {
use gstreamer::MessageView;
match msg.view() {
MessageView::Eos(..) => break,
MessageView::Error(err) => {
eprintln!(
"Error from {:?}: {} ({:?})",
err.src().map(|s| s.path_string()),
err.error(),
err.debug()
);
break;
}
_ => (),
}
}
}
#[test]
fn gst_test_manual_pipeline() {
use gstreamer as gst;
use gstreamer::prelude::*;
// Initialize GStreamer
gst::init().unwrap();
// Create a new pipeline
let pipeline = gst::Pipeline::new();
// Create elements for the pipeline
let src = gst::ElementFactory::make("videotestsrc").build().unwrap();
let sink = gst::ElementFactory::make("autovideosink").build().unwrap();
// Add elements to the pipeline
pipeline.add_many(&[&src, &sink]).unwrap();
// Link elements together
src.link(&sink).unwrap();
// Set the pipeline to the playing state
pipeline.set_state(gst::State::Playing).unwrap();
// Start the main event loop
// let main_loop = glib::MainLoop::new(None, false);
// main_loop.run();
// Shut down the pipeline and GStreamer
let bus = pipeline.bus().unwrap();
let messages = bus.iter_timed(gst::ClockTime::NONE);
for msg in messages {
dbg!(msg);
}
pipeline.set_state(gst::State::Null).unwrap();
// pub fn pipeline_from_str(&self, s: &str) -> Result<Pipeline> {
// let pipeline = gstreamer::parse::launch(s).change_context(Error)?;
// let pipeline = pipeline.downcast::<gstreamer::Pipeline>();
// let pipeline = match pipeline {
// Err(_e) => return Err(Error).attach("Failed to downcast to Pipeline"),
// Ok(p) => p,
// };
// Ok(Pipeline { inner: pipeline })
// }
}

10
gst/src/pad.rs
View File

@@ -20,6 +20,16 @@ impl Pad {
inner: ghost_pad.upcast(),
})
}
pub fn link(&self, peer: &Pad) -> Result<()> {
use gstreamer::prelude::*;
self.inner
.link(&peer.inner)
.change_context(Error)
.attach("Failed to link pads")?;
Ok(())
}
pub fn activate(&self, activate: bool) -> Result<()> {
use gstreamer::prelude::*;
self.inner

141
gst/src/pipeline.rs Normal file
View File

@@ -0,0 +1,141 @@
use crate::{playback::Playbin3, priv_prelude::*};
use gstreamer::State;
#[repr(transparent)]
pub struct Pipeline {
inner: gstreamer::Pipeline,
}
impl core::fmt::Debug for Pipeline {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
f.debug_struct("Pipeline")
.field("pipeline", &self.inner)
// .field("state", &self.pipeline.state(gstreamer::ClockTime::NONE))
.finish()
}
}
impl Drop for Pipeline {
fn drop(&mut self) {
let _ = self.inner.set_state(gstreamer::State::Null);
}
}
impl Pipeline {
pub fn bus(&self) -> Result<Bus> {
let bus = self
.inner
.bus()
.ok_or(Error)
.attach("Failed to get bus from pipeline")?;
Ok(Bus { bus })
}
/// Get the state
pub fn state(
&self,
timeout: impl Into<Option<core::time::Duration>>,
) -> Result<gstreamer::State> {
let (result, current, pending) = self.inner.state(duration_to_clocktime(timeout)?);
result.change_context(Error).attach("Failed to get state")?;
Ok(current)
}
pub fn wait_non_null_sync(&self) -> Result<()> {
if self
.state(core::time::Duration::ZERO)
.change_context(Error)
.attach("Failed to get video context")?
!= gstreamer::State::Null
{
Ok(())
} else {
let bus = self.bus()?;
for message in bus.iter_timed(core::time::Duration::from_secs(1)) {
let view = message.view();
dbg!(&view);
if let gstreamer::MessageView::StateChanged(change) = view
&& change.current() != State::Null
{
break;
}
}
Ok(())
}
}
/// Waits for the pipeline to be ready
pub async fn wait_non_null(&self) -> Result<()> {
if self
.state(None)
.change_context(Error)
.attach("Failed to get video context")?
!= gstreamer::State::Null
{
Ok(())
} else {
use futures::StreamExt;
self.bus()?
.stream()
.filter(|message: &gstreamer::Message| {
let view = message.view();
if let gstreamer::MessageView::StateChanged(change) = view {
core::future::ready(change.current() != gstreamer::State::Null)
} else {
core::future::ready(false)
}
})
.next()
.await;
Ok(())
}
}
pub fn play(&self) -> Result<()> {
self.inner
.set_state(gstreamer::State::Playing)
.change_context(Error)
.attach("Failed to set pipeline to Playing state")?;
Ok(())
}
pub fn pause(&self) -> Result<()> {
self.inner
.set_state(gstreamer::State::Paused)
.change_context(Error)
.attach("Failed to set pipeline to Paused state")?;
Ok(())
}
pub fn ready(&self) -> Result<()> {
self.inner
.set_state(gstreamer::State::Ready)
.change_context(Error)
.attach("Failed to set pipeline to Paused state")?;
Ok(())
}
pub unsafe fn set_state(
&self,
state: gstreamer::State,
) -> Result<gstreamer::StateChangeSuccess> {
let result = self
.inner
.set_state(state)
.change_context(Error)
.attach("Failed to set pipeline state")?;
Ok(result)
}
}
impl core::ops::Deref for Playbin3 {
type Target = Pipeline;
fn deref(&self) -> &Self::Target {
let gp = self
.inner
.downcast_ref::<gstreamer::Pipeline>()
.expect("BUG: Playbin3 must be a pipeline");
unsafe { &*(gp as *const _ as *const Pipeline) }
}
}

127
gst/src/plugins/app/appsink.rs
View File

@@ -1,5 +1,6 @@
use crate::*;
use crate::priv_prelude::*;
#[derive(Debug, Clone)]
pub struct AppSink {
inner: gstreamer::Element,
}
@@ -32,9 +33,9 @@ impl AppSink {
Ok(AppSink { inner })
}
pub fn with_caps(mut self, caps: &gstreamer::Caps) -> Self {
pub fn with_caps(mut self, caps: Caps) -> Self {
use gstreamer::prelude::*;
// self.inner.set_caps(Some(caps));
self.inner.set_property("caps", caps.inner);
self
}
@@ -43,7 +44,7 @@ impl AppSink {
Ok(())
}
pub fn pull_sample(&self, timeout: impl Into<Option<core::time::Duration>>) -> Result<Sample> {
pub fn pull_sample(&self) -> Result<Sample> {
use gstreamer::prelude::*;
self.appsink()
.pull_sample()
@@ -62,7 +63,7 @@ impl AppSink {
.map(From::from))
}
pub fn pull_preroll(&self, timeout: impl Into<Option<core::time::Duration>>) -> Result<Sample> {
pub fn pull_preroll(&self) -> Result<Sample> {
use gstreamer::prelude::*;
self.appsink()
.pull_preroll()
@@ -83,26 +84,106 @@ impl AppSink {
}
}
fn duration_to_clocktime(
timeout: impl Into<Option<core::time::Duration>>,
) -> Result<Option<gstreamer::ClockTime>> {
match timeout.into() {
Some(dur) => {
let clocktime = gstreamer::ClockTime::try_from(dur)
.change_context(Error)
.attach("Failed to convert duration to ClockTime")?;
Ok(Some(clocktime))
}
None => Ok(None),
}
}
pub struct Sample {
inner: gstreamer::Sample,
}
impl From<gstreamer::Sample> for Sample {
fn from(inner: gstreamer::Sample) -> Self {
Sample { inner }
}
}
#[repr(transparent)]
pub struct Sample {
inner: gstreamer::Sample,
}
use gstreamer::BufferRef;
impl Sample {
pub fn buffer(&self) -> Option<&BufferRef> {
self.inner.buffer()
}
pub fn caps(&self) -> Option<&gstreamer::CapsRef> {
self.inner.caps()
}
}
#[test]
fn test_appsink() {
use gstreamer::prelude::*;
use tracing_subscriber::prelude::*;
tracing_subscriber::registry()
.with(
tracing_subscriber::fmt::layer()
.with_thread_ids(true)
.with_file(true),
)
.init();
tracing::info!("Linking videoconvert to appsink");
Gst::new();
let playbin3 = playback::Playbin3::new("pppppppppppppppppppppppppppppp").unwrap().with_uri("https://jellyfin.tsuba.darksailor.dev/Items/6010382cf25273e624d305907010d773/Download?api_key=036c140222464878862231ef66a2bc9c");
let video_convert = plugins::videoconvertscale::VideoConvert::new("vcvcvcvcvcvcvcvcvcvcvcvcvc")
.expect("Create videoconvert");
let appsink = app::AppSink::new("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
.expect("Create appsink")
.with_caps(
Caps::builder(CapsType::Video)
.field("format", "RGB")
.build(),
);
let mut video_sink = video_convert
.link(&appsink)
.expect("Link videoconvert to appsink");
let playbin3 = playbin3.with_video_sink(&video_sink);
let bus = playbin3.bus().unwrap();
// playbin3.play().expect("Play playbin3");
std::thread::spawn({
let playbin3 = playbin3.clone();
move || {
loop {
std::thread::sleep(std::time::Duration::from_secs(5));
playbin3.play();
playbin3.wait_non_null_sync();
let sample = appsink
.try_pull_sample(core::time::Duration::from_secs(5))
.expect("Pull sample from appsink")
.expect("No sample received from appsink");
dbg!(sample.caps());
playbin3.play();
tracing::info!("Played");
}
}
});
for msg in bus.iter_timed(None) {
match msg.view() {
gstreamer::MessageView::Eos(..) => {
tracing::info!("End of stream reached");
break;
}
gstreamer::MessageView::Error(err) => {
tracing::error!(
"Error from {:?}: {} ({:?})",
err.src().map(|s| s.path_string()),
err.error(),
err.debug()
);
break;
}
gstreamer::MessageView::StateChanged(state) => {
eprintln!(
"State changed from {:?} to \x1b[33m{:?}\x1b[0m for {:?}",
state.old(),
state.current(),
state.src().map(|s| s.path_string())
);
}
_ => {}
}
// tracing::info!("{:#?}", &msg.view());
}
// std::thread::sleep(std::time::Duration::from_secs(5));
}

154
gst/src/plugins/playback/playbin3.rs
View File

@@ -1,6 +1,9 @@
use crate::*;
use crate::priv_prelude::*;
#[derive(Debug, Clone)]
#[repr(transparent)]
pub struct Playbin3 {
inner: gstreamer::Element,
pub(crate) inner: gstreamer::Element,
}
impl Drop for Playbin3 {
@@ -61,65 +64,92 @@ impl Playbin3 {
self.inner.property::<f64>("volume")
}
pub fn play(&self) -> Result<()> {
use gstreamer::prelude::*;
self.inner
.set_state(gstreamer::State::Playing)
.change_context(Error)
.attach("Failed to set playbin3 to Playing state")?;
Ok(())
}
pub fn bus(&self) -> Result<Bus> {
let bus = self
.inner
.bus()
.ok_or(Error)
.attach("Failed to get bus from playbin3")?;
Ok(Bus { bus })
}
// pub fn play(&self) -> Result<()> {
// use gstreamer::prelude::*;
// self.inner
// .set_state(gstreamer::State::Playing)
// .change_context(Error)
// .attach("Failed to set playbin3 to Playing state")?;
// Ok(())
// }
//
// pub fn pause(&self) -> Result<()> {
// use gstreamer::prelude::*;
// self.inner
// .set_state(gstreamer::State::Paused)
// .change_context(Error)
// .attach("Failed to set playbin3 to Paused state")?;
// Ok(())
// }
//
// pub fn bus(&self) -> Result<Bus> {
// let bus = self
// .inner
// .bus()
// .ok_or(Error)
// .attach("Failed to get bus from playbin3")?;
// Ok(Bus { bus })
// }
}
#[test]
fn test_playbin3() {
use gstreamer::prelude::*;
use tracing_subscriber::prelude::*;
tracing_subscriber::registry()
.with(
tracing_subscriber::fmt::layer()
.with_thread_ids(true)
.with_file(true),
)
.init();
tracing::info!("Linking videoconvert to appsink");
gstreamer::init().unwrap();
let playbin3 = Playbin3::new("test_playbin3").unwrap().with_uri("https://jellyfin.tsuba.darksailor.dev/Items/6010382cf25273e624d305907010d773/Download?api_key=036c140222464878862231ef66a2bc9c");
// let mut video_sink = Bin::new("wgpu_video_sink");
//
// let video_convert = plugins::videoconvertscale::VideoConvert::new("wgpu_video_convert")
// .expect("Create videoconvert");
// let appsink = AppSink::new("test_appsink").expect("Create appsink");
let appsink = plugins::autodetect::AutoVideoSink::new("test_autodetect_video_sink")
.expect("Create autodetect video sink");
// video_convert
// .link(&appsink)
// .expect("Link videoconvert to appsink");
//
// let sink_pad = video_convert.sink_pad();
// let sink_pad = Pad::ghost(&sink_pad).expect("Create ghost pad from videoconvert src");
// video_sink
// .add(appsink)
// .expect("Add appsink to video sink")
// .add(video_convert)
// .expect("Add videoconvert to video sink")
// .add_pad(&sink_pad)
// .expect("Add ghost pad to video sink");
// sink_pad.activate(true).expect("Activate ghost pad");
let playbin3 = playbin3.with_video_sink(&appsink);
playbin3.play().unwrap();
let bus = playbin3.bus().unwrap();
for msg in bus.iter_timed(None) {
tracing::info!("{:#?}", &msg.view());
}
// std::thread::sleep(std::time::Duration::from_secs(5));
}
// #[test]
// fn test_playbin3() {
// use gstreamer::prelude::*;
// use tracing_subscriber::prelude::*;
// tracing_subscriber::registry()
// .with(
// tracing_subscriber::fmt::layer()
// .with_thread_ids(true)
// .with_file(true),
// )
// .init();
// tracing::info!("Linking videoconvert to appsink");
// Gst::new();
// let playbin3 = Playbin3::new("pppppppppppppppppppppppppppppp").unwrap().with_uri("https://jellyfin.tsuba.darksailor.dev/Items/6010382cf25273e624d305907010d773/Download?api_key=036c140222464878862231ef66a2bc9c");
//
// let video_convert = plugins::videoconvertscale::VideoConvert::new("vcvcvcvcvcvcvcvcvcvcvcvcvc")
// .expect("Create videoconvert");
// let appsink =
// autodetect::AutoVideoSink::new("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa").expect("Create appsink");
//
// let mut video_sink = video_convert
// .link(&appsink)
// .expect("Link videoconvert to appsink");
//
// let playbin3 = playbin3.with_video_sink(&video_sink);
// let bus = playbin3.bus().unwrap();
// playbin3.play().expect("Play playbin3");
// std::thread::spawn(move || loop {
// std::thread::sleep(std::time::Duration::from_secs(15));
// playbin3.play();
// tracing::info!("Played");
// });
// for msg in bus.iter_timed(None) {
// match msg.view() {
// gstreamer::MessageView::Eos(..) => {
// tracing::info!("End of stream reached");
// break;
// }
// gstreamer::MessageView::Error(err) => {
// tracing::error!(
// "Error from {:?}: {} ({:?})",
// err.src().map(|s| s.path_string()),
// err.error(),
// err.debug()
// );
// break;
// }
// gstreamer::MessageView::StateChanged(state) => {
// eprintln!(
// "State changed from {:?} to \x1b[33m{:?}\x1b[0m for {:?}",
// state.old(),
// state.current(),
// state.src().map(|s| s.path_string())
// );
// }
// _ => {}
// }
// // tracing::info!("{:#?}", &msg.view());
// }
// // std::thread::sleep(std::time::Duration::from_secs(5));
// }

1
gst/src/plugins/videoconvertscale/videoconvert.rs
View File

@@ -3,6 +3,7 @@ use crate::*;
pub use gstreamer_video::VideoFormat;
#[repr(transparent)]
#[derive(Debug, Clone)]
pub struct VideoConvert {
inner: gstreamer::Element,
}

5
gst/src/wgpu.rs
View File

@@ -1,3 +1,2 @@
// pub fn copy_sample_to_texture() {
//
// }

11
ui-iced/Cargo.toml
View File

@@ -9,7 +9,16 @@ api = { version = "0.1.0", path = "../api" }
blurhash = "0.2.3"
bytes = "1.11.0"
gpui_util = "0.2.2"
iced = { workspace = true }
iced = { workspace = true, default-features = true, features = [
"advanced",
"canvas",
"image",
"sipper",
"tokio",
"debug",
] }
iced_video_player = { workspace = true }
reqwest = "0.12.24"
tap = "1.0.1"