use std::sync::Arc; use gstreamer as gst; use gstreamer_app as gst_app; use anyhow::{Context, Result}; use winit::{ application::ApplicationHandler, event::*, event_loop::{ActiveEventLoop, EventLoop}, keyboard::*, window::Window, }; pub struct App { state: Option, } impl App { pub fn new() -> Self { Self { state: None } } } pub trait HdrTextureFormatExt { fn is_hdr_format(&self) -> bool; } impl HdrTextureFormatExt for wgpu::TextureFormat { fn is_hdr_format(&self) -> bool { matches!( self, wgpu::TextureFormat::Rgba16Float // | wgpu::TextureFormat::Rg11b10float // | wgpu::TextureFormat::R11g11b10float | wgpu::TextureFormat::Rgb10a2Unorm | wgpu::TextureFormat::Rgba32Float ) } } pub struct State { window: Arc, gst: Video, surface: wgpu::Surface<'static>, video_texture: wgpu::Texture, device: wgpu::Device, queue: wgpu::Queue, config: wgpu::SurfaceConfiguration, pipeline: wgpu::RenderPipeline, bind_group: wgpu::BindGroup, is_surface_initialized: bool, } impl State { async fn new(window: Arc) -> Result { let instance = wgpu::Instance::default(); let surface = instance .create_surface(window.clone()) .context("Failed to create wgpu surface")?; let adapter = instance .request_adapter(&wgpu::RequestAdapterOptions { power_preference: wgpu::PowerPreference::LowPower, compatible_surface: Some(&surface), force_fallback_adapter: false, }) .await .context("Failed to request wgpu adapter")?; let (device, queue) = adapter .request_device(&wgpu::DeviceDescriptor { label: None, required_features: wgpu::Features::empty(), required_limits: wgpu::Limits::default(), memory_hints: wgpu::MemoryHints::default(), ..Default::default() }) .await .context("Failed to request wgpu device")?; let surface_caps = surface.get_capabilities(&adapter); dbg!(&surface_caps); let surface_format = surface_caps .formats .iter() .rev() // float one comes first .find(|f| f.is_hdr_format()) .expect("HDR format not supported") .clone(); let size = window.inner_size(); let config = wgpu::SurfaceConfiguration { usage: wgpu::TextureUsages::RENDER_ATTACHMENT, format: surface_format, width: size.width, height: size.height, present_mode: surface_caps.present_modes[0], alpha_mode: surface_caps.alpha_modes[0], view_formats: vec![], desired_maximum_frame_latency: 2, // calculate upto 5 frames ahead }; surface.configure(&device, &config); let shader = device.create_shader_module(wgpu::include_wgsl!("shader.wgsl")); let texture_bind_group_layout = device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { label: Some("texture_bind_group_layout"), entries: &[ wgpu::BindGroupLayoutEntry { binding: 0, visibility: wgpu::ShaderStages::FRAGMENT, ty: wgpu::BindingType::Texture { multisampled: false, view_dimension: wgpu::TextureViewDimension::D2, sample_type: wgpu::TextureSampleType::Float { filterable: true }, }, count: None, }, wgpu::BindGroupLayoutEntry { binding: 1, visibility: wgpu::ShaderStages::FRAGMENT, ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), count: None, }, ], }); let render_pipeline_layout = device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { label: Some("Jello Render Pipeline Layout"), bind_group_layouts: &[&texture_bind_group_layout], push_constant_ranges: &[], }); let render_pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { label: Some("Jello Render Pipeline"), layout: Some(&render_pipeline_layout), vertex: wgpu::VertexState { module: &shader, entry_point: Some("vs_main"), buffers: &[], compilation_options: wgpu::PipelineCompilationOptions::default(), }, fragment: Some(wgpu::FragmentState { module: &shader, entry_point: Some("fs_main"), compilation_options: wgpu::PipelineCompilationOptions::default(), targets: &[Some(wgpu::ColorTargetState { format: surface_format, blend: Some(wgpu::BlendState::REPLACE), write_mask: wgpu::ColorWrites::ALL, })], }), primitive: wgpu::PrimitiveState::default(), depth_stencil: None, multisample: wgpu::MultisampleState { count: 1, mask: !0, alpha_to_coverage_enabled: false, }, multiview: None, cache: None, }); let texture_size = wgpu::Extent3d { width: size.width, height: size.height, depth_or_array_layers: 1, }; let video_texture = device.create_texture(&wgpu::TextureDescriptor { size: texture_size, mip_level_count: 1, sample_count: 1, dimension: wgpu::TextureDimension::D2, format: surface_format, usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, label: Some("Jello Video Texture"), view_formats: &[], }); // TODO: Use a better sampler let sampler = device.create_sampler(&wgpu::SamplerDescriptor { label: Some("texture_sampler"), address_mode_u: wgpu::AddressMode::ClampToEdge, address_mode_v: wgpu::AddressMode::ClampToEdge, address_mode_w: wgpu::AddressMode::ClampToEdge, mag_filter: wgpu::FilterMode::Linear, min_filter: wgpu::FilterMode::Linear, mipmap_filter: wgpu::FilterMode::Nearest, ..Default::default() }); let bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor { layout: &texture_bind_group_layout, entries: &[ wgpu::BindGroupEntry { binding: 0, resource: wgpu::BindingResource::TextureView( &video_texture.create_view(&wgpu::TextureViewDescriptor::default()), ), }, wgpu::BindGroupEntry { binding: 1, resource: wgpu::BindingResource::Sampler(&sampler), }, ], label: Some("Jello Texture Bind Group"), }); let gst = Video::new().context("Failed to create Video")?; // surface.configure(&device, &config); Ok(Self { window, gst, surface, video_texture, device, queue, config, is_surface_initialized: true, bind_group, pipeline: render_pipeline, }) } // async fn next_frame(&mut self) fn resize(&mut self, width: u32, height: u32) { if width > 0 && height > 0 { self.config.width = width; self.config.height = height; self.surface.configure(&self.device, &self.config); self.is_surface_initialized = true; } } fn render(&mut self) -> Result<(), wgpu::SurfaceError> { if !self.is_surface_initialized { return Ok(()); } self.gst.poll(); self.copy_next_frame_to_texture() .inspect_err(|e| { tracing::error!("Failed to copy video frame to texture: {e:?}"); }) .map_err(|_| wgpu::SurfaceError::Lost)?; let output = match self.surface.get_current_texture() { Ok(output) => output, Err(wgpu::SurfaceError::Lost) => { self.surface.configure(&self.device, &self.config); return Ok(()); } Err(e) => return Err(e), }; let view = output .texture .create_view(&wgpu::TextureViewDescriptor::default()); let mut encoder = self .device .create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("Jello Render Encoder"), }); let mut render_pass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor { label: Some("Jello Render Pass"), color_attachments: &[Some(wgpu::RenderPassColorAttachment { view: &view, resolve_target: None, ops: wgpu::Operations { load: wgpu::LoadOp::Clear(wgpu::Color { r: 0.1, g: 0.2, b: 0.3, a: 1.0, }), store: wgpu::StoreOp::Store, }, depth_slice: None, })], depth_stencil_attachment: None, occlusion_query_set: None, timestamp_writes: None, }); render_pass.set_pipeline(&self.pipeline); render_pass.set_bind_group(0, &self.bind_group, &[]); render_pass.draw(0..3, 0..1); drop(render_pass); self.queue.submit(std::iter::once(encoder.finish())); output.present(); self.window.request_redraw(); Ok(()) } pub fn copy_next_frame_to_texture(&mut self) -> Result<()> { let frame = self .gst .appsink .try_pull_sample(gst::ClockTime::NONE) .context("Failed to pull sample from appsink")?; let caps = frame.caps().context("Failed to get caps from sample")?; let size = caps .structure(0) .context("Failed to get structure from caps")?; let width = size .get::("width") .context("Failed to get width from caps")? as u32; let height = size .get::("height") .context("Failed to get height from caps")? as u32; let texture_size = self.video_texture.size(); if texture_size.width != width || texture_size.height != height { tracing::info!( "Resizing video texture from {}x{} to {}x{}", texture_size.width, texture_size.height, width, height ); self.video_texture = self.device.create_texture(&wgpu::TextureDescriptor { size: wgpu::Extent3d { width: width as u32, height: height as u32, depth_or_array_layers: 1, }, mip_level_count: 1, sample_count: 1, dimension: wgpu::TextureDimension::D2, format: self.config.format, usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST, label: Some("Jello Video Texture"), view_formats: &[], }); let texture_bind_group_layout = self.device .create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { label: Some("texture_bind_group_layout"), entries: &[ wgpu::BindGroupLayoutEntry { binding: 0, visibility: wgpu::ShaderStages::FRAGMENT, ty: wgpu::BindingType::Texture { multisampled: false, view_dimension: wgpu::TextureViewDimension::D2, sample_type: wgpu::TextureSampleType::Float { filterable: true, }, }, count: None, }, wgpu::BindGroupLayoutEntry { binding: 1, visibility: wgpu::ShaderStages::FRAGMENT, ty: wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering), count: None, }, ], }); let sampler = self.device.create_sampler(&wgpu::SamplerDescriptor { label: Some("texture_sampler"), address_mode_u: wgpu::AddressMode::ClampToEdge, address_mode_v: wgpu::AddressMode::ClampToEdge, address_mode_w: wgpu::AddressMode::ClampToEdge, mag_filter: wgpu::FilterMode::Linear, min_filter: wgpu::FilterMode::Linear, mipmap_filter: wgpu::FilterMode::Nearest, ..Default::default() }); self.bind_group = self.device.create_bind_group(&wgpu::BindGroupDescriptor { layout: &texture_bind_group_layout, entries: &[ wgpu::BindGroupEntry { binding: 0, resource: wgpu::BindingResource::TextureView( &self .video_texture .create_view(&wgpu::TextureViewDescriptor::default()), ), }, wgpu::BindGroupEntry { binding: 1, resource: wgpu::BindingResource::Sampler(&sampler), }, ], label: Some("Jello Texture Bind Group"), }); } let texture = &self.video_texture; let buffer = frame.buffer().context("Failed to get buffer from sample")?; let map = buffer .map_readable() .context("Failed to map buffer readable")?; self.queue.write_texture( wgpu::TexelCopyTextureInfo { texture: &texture, mip_level: 0, origin: wgpu::Origin3d::ZERO, aspect: wgpu::TextureAspect::All, }, &map, wgpu::TexelCopyBufferLayout { offset: 0, bytes_per_row: Some(4 * width as u32), rows_per_image: Some(height as u32), }, texture.size(), ); drop(map); // drop(buffer); drop(frame); Ok(()) } } impl ApplicationHandler for App { fn resumed(&mut self, event_loop: &ActiveEventLoop) { #[allow(unused_mut)] let mut window_attributes = Window::default_attributes(); let window = Arc::new(event_loop.create_window(window_attributes).unwrap()); let monitor = event_loop .primary_monitor() .or_else(|| window.current_monitor()); // window.set_fullscreen(None); window.set_fullscreen(Some(winit::window::Fullscreen::Borderless(monitor))); self.state = Some(pollster::block_on(State::new(window)).expect("Failed to block")); } fn user_event(&mut self, _event_loop: &ActiveEventLoop, event: State) { self.state = Some(event); } fn about_to_wait(&mut self, _event_loop: &ActiveEventLoop) { let state = match &mut self.state { Some(state) => state, None => return, }; state.window.request_redraw(); } fn window_event( &mut self, event_loop: &ActiveEventLoop, _window_id: winit::window::WindowId, event: WindowEvent, ) { let state = match &mut self.state { Some(canvas) => canvas, None => return, }; match event { WindowEvent::CloseRequested => event_loop.exit(), WindowEvent::Resized(size) => { tracing::info!("Window resized to {size:?}"); state.resize(size.width, size.height) } WindowEvent::RedrawRequested => { // if state.gst.poll() { // event_loop.exit(); // return; // } match state.render() { Ok(_) => {} // Reconfigure the surface if lost Err(wgpu::SurfaceError::Lost | wgpu::SurfaceError::Outdated) => { let size = state.window.inner_size(); tracing::info!("Reconfiguring surface to {size:?}"); state.resize(size.width, size.height); } // The system is out of memory, we should probably quit Err(wgpu::SurfaceError::OutOfMemory) => event_loop.exit(), // All other errors (Outdated, Timeout) should be resolved by the next frame Err(e) => { tracing::error!("Failed to render frame: {e:?}"); } } } // WindowEvent::AboutToWait => { // state.window.request_redraw(); // } WindowEvent::KeyboardInput { event: KeyEvent { physical_key: PhysicalKey::Code(code), state, .. }, .. } => match (code, state.is_pressed()) { (KeyCode::Escape, true) => event_loop.exit(), (KeyCode::KeyQ, true) => event_loop.exit(), _ => {} }, _ => {} } } } pub fn main() -> anyhow::Result<()> { tracing_subscriber::fmt::init(); let event_loop = EventLoop::with_user_event().build()?; let mut app = App::new(); event_loop.run_app(&mut app)?; Ok(()) } pub struct Video { pipeline: gst::Pipeline, bus: gst::Bus, appsink: gst_app::AppSink, } impl Video { pub fn new() -> Result { gst::init()?; use gst::prelude::*; let pipeline = gst::parse::launch( r##"playbin3 uri=https://jellyfin.tsuba.darksailor.dev/Items/6010382cf25273e624d305907010d773/Download?api_key=036c140222464878862231ef66a2bc9c video-sink="videoconvert ! video/x-raw,format=RGB10A2_LE ! appsink name=appsink""##, ).context("Failed to parse gst pipeline")?; let pipeline = pipeline .downcast::() .map_err(|_| anyhow::anyhow!("Failed to downcast gst element to Pipeline"))?; let video_sink = pipeline.property::("video-sink"); let appsink = video_sink .by_name("appsink") .context("Failed to get appsink from video-sink")? .downcast::() .map_err(|_| { anyhow::anyhow!("Failed to downcast video-sink appsink to gst_app::AppSink") })?; // appsink.set_property("max-buffers", 2u32); // appsink.set_property("emit-signals", true); appsink.set_callbacks( gst_app::AppSinkCallbacks::builder() .new_sample(|_appsink| Ok(gst::FlowSuccess::Ok)) .build(), ); let bus = pipeline.bus().context("Failed to get gst pipeline bus")?; pipeline.set_state(gst::State::Playing)?; pipeline .state(gst::ClockTime::from_seconds(5)) .0 .context("Failed to wait for pipeline")?; Ok(Self { pipeline, bus, appsink, }) } pub fn poll(&mut self) -> bool { use gst::prelude::*; for msg in self.bus.iter_timed(gst::ClockTime::ZERO) { use gst::MessageView; match msg.view() { MessageView::Eos(..) => { tracing::info!("End of stream"); self.pipeline.set_state(gst::State::Null).ok(); return true; } MessageView::Error(err) => { tracing::error!( "Error from {:?}: {} ({:?})", err.src().map(|s| s.path_string()), err.error(), err.debug() ); self.pipeline.set_state(gst::State::Null).ok(); return true; } _ => {} } } false } }