diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 54adb05..0cbbd57 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,7 +39,6 @@ jobs: - Windows - driver only - gateway only - - legacy tokio include: - name: beta @@ -75,6 +74,16 @@ jobs: sudo apt-get update sudo apt-get install -y libopus-dev + - name: Install yt-dlp (Unix) + if: runner.os != 'Windows' + run: | + sudo wget https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp -O /usr/local/bin/yt-dlp + sudo chmod a+rx /usr/local/bin/yt-dlp + + - name: Install yt-dlp (Windows) + if: runner.os == 'Windows' + run: choco install yt-dlp + - name: Setup cache if: runner.os != 'macOS' uses: actions/cache@v2 @@ -175,9 +184,9 @@ jobs: - name: 'Build serenity/voice_receive' working-directory: examples run: cargo build -p voice_receive - - name: 'Build serenity/voice_storage' + - name: 'Build serenity/voice_cached_audio' working-directory: examples - run: cargo build -p voice_storage + run: cargo build -p voice_cached_audio - name: 'Build twilight' working-directory: examples run: cargo build -p twilight diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index ced2cdd..d14ac6e 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -29,12 +29,14 @@ Songbird's **driver** is a mixed sync/async system for running voice connections Audio processing remains synchronous for the following reasons: * Encryption, encoding, and mixing are compute bound tasks which cannot be subdivided cleanly by the Tokio executor. Having these block the scheduler's finite thread count has a significant impact on servicing other tasks. * `Read` and `Seek` are considerably more user-friendly to use, implement, and integrate than `AsyncRead`, `AsyncBufRead`, and `AsyncSeek`. +* Symphonia implements all of its functionality based on synchronous I/O. ## Tasks Songbird subdivides voice connection handling into several long- and short-lived tasks. * **Core**: Handles and directs commands received from the driver. Responsible for connection/reconnection, and creates network tasks. * **Mixer**: Combines audio sources together, Opus encodes the result, and encrypts the built packets every 20ms. Responsible for handling track commands/state. ***Synchronous***. +* **Thread Pool**: A dynamically sized thread-pool for I/O tasks. Creates lazy tracks using `Compose` if sync creation is needed, otherwise spawns a tokio task. Seek operations always go to the thread pool. ***Synchronous***. * **Disposer**: Used by mixer thread to dispose of data with potentially long/blocking `Drop` implementations (i.e., audio sources). ***Synchronous***. * **Events**: Stores and runs event handlers, tracks event timing, and handles * **Websocket**: *Network task.* Sends speaking status updates and keepalives to Discord, and receives client (dis)connect events. @@ -52,23 +54,22 @@ src/driver/* ## Audio handling ### Input -Inputs are raw audio sources: composed of a `Reader` (which can be `Read`-only or `Read + Seek`), a framing mechanism, and a codec. -Several wrappers exist to add `Seek` capabilities to one-way streams via storage or explicitly recreating the struct. +Inputs are audio sources supporting lazy initialisation, being either: +* **lazy inputs**—a trait object which allows an instructions to create an audio source to be cheaply stored. This will be initialised when needed either synchronously or asynchronously based on what which methods the trait object supports. +* **live inputs**—a usable audio object implementing `MediaSource: Read + Seek`. `Seek` support may be dummied in, as seek use and support is gated by `MediaSource`. These can be passed in at various stages of processing by symphonia. -Framing is not always needed (`Raw`), but makes it possible to consume the correct number of bytes needed to decode one audio packet (and/or simplify skipping through the stream). -Currently, Opus and raw (`i16`/`f32`) audio sources are supported, though only the DCA framing for Opus is implemented. -At present, the use of the FFmpeg executable allows us to receive raw input, but at heavy memory cost. -Further implementations are possible in the present framework (e.g., WebM/MKV and Ogg containers, MP3 and linked FFI FFmpeg as codecs). +Several wrappers exist to add `Seek` capabilities to one-way streams via storage or explicitly recreating the struct, `AsyncRead` adapters, and raw audio input adapters. Internally, the mixer uses floating-point audio to prevent clipping and allow more granular volume control. -If a source is known to use the Opus codec (and is the only source), then it can bypass mixing altogether. +Symphonia is used to demux and decode input files in a variety of formats into this floating-point buffer: songbird supports all codecs and containers which are part of the symphonia project, while adding support for Opus decoding and DCA1 container files. +If a source uses the Opus codec (and is the only source), then it can bypass mixing and re-encoding altogether, saving CPU cycles per server. ``` src/input/* ``` ### Tracks -Tracks hold additional state which is expected to change over the lifetime of a track: position, play state, and modifiers like volume. +Tracks hold additional state which is expected to change over the lifetime of a track: position, play state, and modifiers like volume. Tracks (and their handles) also allow per-source events to be inserted. Tracks are defined in user code, where they are fully modifiable, before being passed into the driver. diff --git a/Cargo.toml b/Cargo.toml index 94150e1..56e2fba 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ authors = ["Kyle Simpson "] description = "An async Rust library for the Discord voice API." documentation = "https://docs.rs/songbird" -edition = "2018" +edition = "2021" homepage = "https://github.com/serenity-rs/songbird" include = ["src/**/*.rs", "Cargo.toml", "build.rs"] keywords = ["discord", "api", "rtp", "audio"] @@ -10,15 +10,20 @@ license = "ISC" name = "songbird" readme = "README.md" repository = "https://github.com/serenity-rs/songbird.git" -version = "0.3.0" +version = "0.2.2" +rust-version = "1.61" [dependencies] derivative = "2" +pin-project = "1" serde = { version = "1", features = ["derive"] } serde_json = "1" tracing = { version = "0.1", features = ["log"] } tracing-futures = "0.2" -symphonia-core = "0.5" + +[dependencies.once_cell] +version = "1" +optional = true [dependencies.async-trait] optional = true @@ -45,9 +50,8 @@ version = "5" [dependencies.discortp] features = ["discord-full"] optional = true -version = "0.4" +version = "0.5" -# Temporary hack to pin MSRV. [dependencies.flume] optional = true version = "0.10" @@ -55,18 +59,41 @@ version = "0.10" [dependencies.futures] version = "0.3" +[dependencies.lazy_static] +optional = true +version = "1" + [dependencies.parking_lot] optional = true version = "0.12" -[dependencies.pin-project] -optional = true -version = "1" - [dependencies.rand] optional = true version = "0.8" +[dependencies.reqwest] +optional = true +default-features = false +features = ["stream"] +version = "0.11" + +[dependencies.ringbuf] +optional = true +version = "0.2" + +[dependencies.rubato] +optional = true +version = "0.12" + +[dependencies.rusty_pool] +optional = true +version = "0.7" + +[dependencies.serde-aux] +default-features = false +optional = true +version = "3" + [dependencies.serenity] optional = true version = "0.11" @@ -81,11 +108,29 @@ version = "0.1" optional = true version = "1" +[dependencies.symphonia] +optional = true +default-features = false +version = "0.5" +git = "https://github.com/FelixMcFelix/Symphonia" +branch = "songbird-fixes" + +[dependencies.symphonia-core] +optional = true +version = "0.5" +git = "https://github.com/FelixMcFelix/Symphonia" +branch = "songbird-fixes" + [dependencies.tokio] optional = true version = "1.0" default-features = false +[dependencies.tokio-util] +optional = true +version = "0.7" +features = ["io"] + [dependencies.twilight-gateway] optional = true version = "0.12.0" @@ -106,7 +151,7 @@ version = "2" [dependencies.uuid] optional = true -version = "0.8" +version = "1" features = ["v4"] [dependencies.xsalsa20poly1305] @@ -116,7 +161,10 @@ features = ["std"] [dev-dependencies] criterion = "0.3" +ntest = "0.8" +symphonia = { version = "0.5", features = ["mp3"], git = "https://github.com/FelixMcFelix/Symphonia", branch = "songbird-fixes" } utils = { path = "utils" } +tokio = { version = "1", features = ["rt", "rt-multi-thread"] } [features] # Core features @@ -126,19 +174,33 @@ default = [ "gateway", ] gateway = [ - "gateway-core", + "dashmap", + "flume", + "once_cell", + "parking_lot", "tokio/sync", "tokio/time", ] -gateway-core = [ - "dashmap", - "flume", - "parking_lot", - "pin-project", -] driver = [ + "async-trait", "async-tungstenite", - "driver-core", + "audiopus", + "byteorder", + "discortp", + "reqwest", + "flume", + "lazy_static", + "parking_lot", + "rand", + "ringbuf", + "rubato", + "serde-aux", + "serenity-voice-model", + "streamcatcher", + "symphonia", + "symphonia-core", + "rusty_pool", + "tokio-util", "tokio/fs", "tokio/io-util", "tokio/macros", @@ -147,24 +209,13 @@ driver = [ "tokio/rt", "tokio/sync", "tokio/time", -] -driver-core = [ - "async-trait", - "audiopus", - "byteorder", - "discortp", - "flume", - "parking_lot", - "rand", - "serenity-voice-model", - "streamcatcher", "typemap_rev", "url", "uuid", "xsalsa20poly1305", ] -rustls = ["async-tungstenite/tokio-rustls-webpki-roots", "rustls-marker"] -native = ["async-tungstenite/tokio-native-tls", "native-marker"] +rustls = ["async-tungstenite/tokio-rustls-webpki-roots", "reqwest/rustls-tls", "rustls-marker"] +native = ["async-tungstenite/tokio-native-tls", "native-marker", "reqwest/native-tls"] serenity-rustls = ["serenity/rustls_backend", "rustls", "gateway", "serenity-deps"] serenity-native = ["serenity/native_tls_backend", "native", "gateway", "serenity-deps"] twilight-rustls = ["twilight", "twilight-gateway/rustls-native-roots", "rustls", "gateway"] @@ -178,8 +229,6 @@ rustls-marker = [] native-marker = [] # Behaviour altering features. -youtube-dlc = [] -yt-dlp = [] builtin-queue = [] # Used for docgen/testing/benchmarking. @@ -189,6 +238,7 @@ internals = [] [[bench]] name = "base-mixing" path = "benches/base-mixing.rs" +required-features = ["internals"] harness = false [[bench]] diff --git a/Makefile.toml b/Makefile.toml index 4199101..7e0e958 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -26,6 +26,10 @@ dependencies = ["format"] [tasks.build-variants] dependencies = ["build", "build-gateway", "build-driver"] +[tasks.check] +args = ["check", "--features", "full-doc"] +dependencies = ["format"] + [tasks.clippy] args = ["clippy", "--features", "full-doc", "--", "-D", "warnings"] dependencies = ["format"] diff --git a/README.md b/README.md index e3ad164..bd54340 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![docs-badge][]][docs] [![build badge]][build] [![guild-badge][]][guild] [![crates.io version]][crates.io link] [![rust 1.60.0+ badge]][rust 1.60.0+ link] +[![docs-badge][]][docs] [![build badge]][build] [![guild-badge][]][guild] [![crates.io version]][crates.io link] [![rust 1.61.0+ badge]][rust 1.61.0+ link] # Songbird @@ -19,27 +19,48 @@ The library offers: ## Intents Songbird's gateway functionality requires you to specify the `GUILD_VOICE_STATES` intent. +## Codec support +Songbird supports all [codecs and formats provided by Symphonia] (pure-Rust), with Opus support +provided by [audiopus] (an FFI wrapper for libopus). + +**By default, *Songbird will not request any codecs from Symphonia*.** To change this, in your own +project you will need to depend on Symphonia as well. + +```toml +# Including songbird alone gives you support for Opus via the DCA file format. +[dependencies.songbird] +version = "0.4" +features = ["builtin-queue"] + +# To get additional codecs, you *must* add Symphonia yourself. +# This includes the default formats (MKV/WebM, Ogg, Wave) and codecs (FLAC, PCM, Vorbis)... +[dependencies.symphonia] +# version = "0.5" +features = ["aac", "mp3", "isomp4", "alac"] # ...as well as any extras you need! +# **NOTE**: For now, please use this fork in pre-releases for a key fix to +# seeking on streamed MKV/WebM files. +git = "https://github.com/FelixMcFelix/Symphonia" +branch = "songbird-fixes" +``` + ## Dependencies Songbird needs a few system dependencies before you can use it. - Opus - Audio codec that Discord uses. -If you are on Windows and you are using the MSVC toolchain, a prebuilt DLL is provided for you, you do not have to do anything. -On other platforms, you will have to install it. You can install the library with `apt install libopus-dev` on Ubuntu or `pacman -S opus` on Arch Linux. +[audiopus] will use installed libopus binaries if available via pkgconf on Linux/MacOS, otherwise you will need to install cmake to build opus from source. +This is always the case on Windows. +For Unix systems, you can install the library with `apt install libopus-dev` on Ubuntu or `pacman -S opus` on Arch Linux. If you do not have it installed it will be built for you. However, you will need a C compiler and the GNU autotools installed. Again, these can be installed with `apt install build-essential autoconf automake libtool m4` on Ubuntu or `pacman -S base-devel` on Arch Linux. This is a required dependency. Songbird cannot work without it. -- FFmpeg - Audio/Video conversion tool. -You can install the tool with `apt install ffmpeg` on Ubuntu or `pacman -S ffmpeg` on Arch Linux. - -This is an optional, but recommended dependency. It allows Songbird to convert from, for instance, .mp4 files to the audio format Discord uses. - -- youtube-dl - Audio/Video download tool. -You can install the tool with Python's package manager, pip, which we recommend for youtube-dl. You can do it with the command `pip install youtube_dl`. +- yt-dlp / youtube-dl / (similar forks) - Audio/Video download tool. +yt-dlp can be installed [according to the installation instructions on the main repo]. +You can install youtube-dl with Python's package manager, pip, which we recommend for youtube-dl. You can do it with the command `pip install youtube_dl`. Alternatively, you can install it with your system's package manager, `apt install youtube-dl` on Ubuntu or `pacman -S youtube-dl` on Arch Linux. -This is an optional dependency. It allows Songbird to download an audio source from the Internet, which will be converted to the audio format Discord uses. +This is an optional dependency for users, but is required as a dev-dependency. It allows Songbird to download audio/video sources from the Internet from a variety of webpages, which it will convert to the Opus audio format Discord uses. ## Examples Full examples showing various types of functionality and integrations can be found in [this crate's examples directory]. @@ -56,6 +77,9 @@ Songbird's logo is based upon the copyright-free image ["Black-Capped Chickadee" [lavalink]: https://github.com/freyacodes/Lavalink [this crate's examples directory]: https://github.com/serenity-rs/songbird/tree/current/examples [our contributor guidelines]: CONTRIBUTING.md +[codecs and formats provided by Symphonia]: https://github.com/pdeljanov/Symphonia#formats-demuxers +[audiopus]: https://github.com/lakelezz/audiopus +[according to the installation instructions on the main repo]: https://github.com/yt-dlp/yt-dlp#installation [build badge]: https://img.shields.io/github/workflow/status/serenity-rs/songbird/CI?style=flat-square [build]: https://github.com/serenity-rs/songbird/actions @@ -69,5 +93,5 @@ Songbird's logo is based upon the copyright-free image ["Black-Capped Chickadee" [crates.io link]: https://crates.io/crates/songbird [crates.io version]: https://img.shields.io/crates/v/songbird.svg?style=flat-square -[rust 1.60.0+ badge]: https://img.shields.io/badge/rust-1.60.0+-93450a.svg?style=flat-square -[rust 1.60.0+ link]: https://blog.rust-lang.org/2022/04/07/Rust-1.60.0.html +[rust 1.61.0+ badge]: https://img.shields.io/badge/rust-1.61.0+-93450a.svg?style=flat-square +[rust 1.61.0+ link]: https://blog.rust-lang.org/2022/05/19/Rust-1.61.0.html diff --git a/benches/base-mixing.rs b/benches/base-mixing.rs index 7828bae..d9e12c3 100644 --- a/benches/base-mixing.rs +++ b/benches/base-mixing.rs @@ -1,29 +1,102 @@ -use criterion::{black_box, criterion_group, criterion_main, BatchSize, Criterion}; -use songbird::{constants::*, input::Input}; +use criterion::{black_box, criterion_group, criterion_main, BatchSize, BenchmarkId, Criterion}; +use songbird::{ + constants::*, + driver::{ + bench_internals::mixer::{mix_logic, state::DecodeState}, + MixMode, + }, + input::{codecs::*, Input, LiveInput, Parsed}, +}; +use std::io::Cursor; +use symphonia_core::audio::{AudioBuffer, Layout, SampleBuffer, Signal, SignalSpec}; pub fn mix_one_frame(c: &mut Criterion) { - let floats = utils::make_sine(STEREO_FRAME_SIZE, true); - let mut raw_buf = [0f32; STEREO_FRAME_SIZE]; + let floats = utils::make_sine(1 * STEREO_FRAME_SIZE, true); - c.bench_function("Mix stereo source", |b| { - b.iter_batched_ref( - || black_box(Input::float_pcm(true, floats.clone().into())), - |input| { - input.mix(black_box(&mut raw_buf), black_box(1.0)); - }, - BatchSize::SmallInput, - ) - }); + let symph_layout = MixMode::Stereo.into(); - c.bench_function("Mix mono source", |b| { - b.iter_batched_ref( - || black_box(Input::float_pcm(false, floats.clone().into())), - |input| { - input.mix(black_box(&mut raw_buf), black_box(1.0)); + let mut symph_mix = AudioBuffer::::new( + MONO_FRAME_SIZE as u64, + symphonia_core::audio::SignalSpec::new_with_layout(SAMPLE_RATE_RAW as u32, symph_layout), + ); + let mut resample_scratch = AudioBuffer::::new( + MONO_FRAME_SIZE as u64, + SignalSpec::new_with_layout(SAMPLE_RATE_RAW as u32, Layout::Stereo), + ); + + let mut group = c.benchmark_group("Stereo Target"); + + for (pres, hz) in [("", 48_000), (" (Resample)", 44_100)] { + group.bench_with_input( + BenchmarkId::new(format!("Stereo Source{}", pres), hz), + &hz, + |b, i| { + b.iter_batched_ref( + || black_box(make_src(&floats, 2, *i)), + |(ref mut input, ref mut local_input)| { + symph_mix.clear(); + symph_mix.render_reserved(Some(MONO_FRAME_SIZE)); + resample_scratch.clear(); + + black_box(mix_logic::mix_symph_indiv( + &mut symph_mix, + &mut resample_scratch, + input, + local_input, + black_box(1.0), + None, + )); + }, + BatchSize::SmallInput, + ) }, - BatchSize::SmallInput, - ) - }); + ); + + group.bench_with_input( + BenchmarkId::new(format!("Mono Source{}", pres), hz), + &hz, + |b, i| { + b.iter_batched_ref( + || black_box(make_src(&floats, 1, *i)), + |(ref mut input, ref mut local_input)| { + symph_mix.clear(); + symph_mix.render_reserved(Some(MONO_FRAME_SIZE)); + resample_scratch.clear(); + + black_box(mix_logic::mix_symph_indiv( + &mut symph_mix, + &mut resample_scratch, + input, + local_input, + black_box(1.0), + None, + )); + }, + BatchSize::SmallInput, + ) + }, + ); + } + + group.finish(); +} + +fn make_src(src: &Vec, chans: u32, hz: u32) -> (Parsed, DecodeState) { + let local_input = Default::default(); + + let adapted: Input = + songbird::input::RawAdapter::new(Cursor::new(src.clone()), hz, chans).into(); + let promoted = match adapted { + Input::Live(l, _) => l.promote(&CODEC_REGISTRY, &PROBE), + _ => panic!("Failed to create a guaranteed source."), + }; + let parsed = match promoted { + Ok(LiveInput::Parsed(parsed)) => parsed, + Err(e) => panic!("AR {:?}", e), + _ => panic!("Failed to create a guaranteed source."), + }; + + (parsed, local_input) } criterion_group!(benches, mix_one_frame); diff --git a/benches/mixing-task.rs b/benches/mixing-task.rs index 99f5b69..48c7a88 100644 --- a/benches/mixing-task.rs +++ b/benches/mixing-task.rs @@ -1,3 +1,5 @@ +use std::error::Error; + use criterion::{ black_box, criterion_group, @@ -11,12 +13,18 @@ use flume::{Receiver, Sender, TryRecvError}; use songbird::{ constants::*, driver::{ - bench_internals::{mixer::Mixer, task_message::*, CryptoState}, + bench_internals::{ + self, + mixer::{state::InputState, Mixer}, + task_message::*, + CryptoState, + }, Bitrate, }, - input::{cached::Compressed, Input}, + input::{cached::Compressed, codecs::*, Input, RawAdapter}, tracks, }; +use std::io::Cursor; use tokio::runtime::{Handle, Runtime}; use xsalsa20poly1305::{aead::NewAead, XSalsa20Poly1305 as Cipher, KEY_SIZE}; @@ -79,14 +87,17 @@ fn mixer_float( let floats = utils::make_sine(10 * STEREO_FRAME_SIZE, true); - let mut tracks = vec![]; for i in 0..num_tracks { - let input = Input::float_pcm(true, floats.clone().into()); - tracks.push(tracks::create_player(input).0.into()); + let input: Input = RawAdapter::new(Cursor::new(floats.clone()), 48_000, 2).into(); + let promoted = match input { + Input::Live(l, _) => l.promote(&CODEC_REGISTRY, &PROBE), + _ => panic!("Failed to create a guaranteed source."), + }; + let (handle, mut ctx) = + bench_internals::track_context(Input::Live(promoted.unwrap(), None).into()); + out.0.add_track(ctx); } - out.0.tracks = tracks; - out } @@ -104,15 +115,18 @@ fn mixer_float_drop( ) { let mut out = dummied_mixer(handle); - let mut tracks = vec![]; for i in 0..num_tracks { let floats = utils::make_sine((i / 5) * STEREO_FRAME_SIZE, true); - let input = Input::float_pcm(true, floats.clone().into()); - tracks.push(tracks::create_player(input).0.into()); + let input: Input = RawAdapter::new(Cursor::new(floats.clone()), 48_000, 2).into(); + let promoted = match input { + Input::Live(l, _) => l.promote(&CODEC_REGISTRY, &PROBE), + _ => panic!("Failed to create a guaranteed source."), + }; + let (handle, mut ctx) = + bench_internals::track_context(Input::Live(promoted.unwrap(), None).into()); + out.0.add_track(ctx); } - out.0.tracks = tracks; - out } @@ -129,22 +143,28 @@ fn mixer_opus( ) { // should add a single opus-based track. // make this fully loaded to prevent any perf cost there. - let mut out = dummied_mixer(handle); + let mut out = dummied_mixer(handle.clone()); let floats = utils::make_sine(6 * STEREO_FRAME_SIZE, true); - let mut tracks = vec![]; + let input: Input = RawAdapter::new(Cursor::new(floats), 48_000, 2).into(); + + let mut src = handle.block_on(async move { + Compressed::new(input, Bitrate::BitsPerSecond(128_000)) + .await + .expect("These parameters are well-defined.") + }); - let mut src = Compressed::new( - Input::float_pcm(true, floats.clone().into()), - Bitrate::BitsPerSecond(128_000), - ) - .expect("These parameters are well-defined."); src.raw.load_all(); - tracks.push(tracks::create_player(src.into()).0.into()); + let promoted = match src.into() { + Input::Live(l, _) => l.promote(&CODEC_REGISTRY, &PROBE), + _ => panic!("Failed to create a guaranteed source."), + }; + let (handle, mut ctx) = + bench_internals::track_context(Input::Live(promoted.unwrap(), None).into()); - out.0.tracks = tracks; + out.0.add_track(ctx); out } diff --git a/examples/Cargo.toml b/examples/Cargo.toml index 77c21b9..9737722 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -1,8 +1,8 @@ [workspace] members = [ "serenity/voice", + "serenity/voice_cached_audio", "serenity/voice_events_queue", "serenity/voice_receive", - "serenity/voice_storage", "twilight", ] diff --git a/examples/serenity/voice/Cargo.toml b/examples/serenity/voice/Cargo.toml index 171adfc..1e8a03a 100644 --- a/examples/serenity/voice/Cargo.toml +++ b/examples/serenity/voice/Cargo.toml @@ -2,12 +2,19 @@ name = "voice" version = "0.1.0" authors = ["my name "] -edition = "2018" +edition = "2021" [dependencies] tracing = "0.1" tracing-subscriber = "0.2" tracing-futures = "0.2" +reqwest = "0.11" + +[dependencies.symphonia] +version = "0.5" +features = ["aac", "mp3", "isomp4", "alac"] +git = "https://github.com/FelixMcFelix/Symphonia" +branch = "songbird-fixes" [dependencies.songbird] path = "../../../" diff --git a/examples/serenity/voice/src/main-skip.rs b/examples/serenity/voice/src/main-skip.rs new file mode 100644 index 0000000..50718d4 --- /dev/null +++ b/examples/serenity/voice/src/main-skip.rs @@ -0,0 +1,417 @@ +//! Requires the "client", "standard_framework", and "voice" features be enabled in your +//! Cargo.toml, like so: +//! +//! ```toml +//! [dependencies.serenity] +//! git = "https://github.com/serenity-rs/serenity.git" +//! features = ["client", "standard_framework", "voice"] +//! ``` +use std::env; + +// This trait adds the `register_songbird` and `register_songbird_with` methods +// to the client builder below, making it easy to install this voice client. +// The voice client can be retrieved in any command using `songbird::get(ctx).await`. +use songbird::SerenityInit; + +// Event related imports to detect track creation failures. +use songbird::events::{Event, EventContext, EventHandler as VoiceEventHandler, TrackEvent}; + +use std::time::Duration; + +// Import the `Context` to handle commands. +use serenity::client::Context; + +use serenity::{ + async_trait, + client::{Client, EventHandler}, + framework::{ + standard::{ + macros::{command, group}, + Args, CommandResult, + }, + StandardFramework, + }, + model::{channel::Message, gateway::Ready}, + prelude::GatewayIntents, + Result as SerenityResult, +}; + +struct Handler; + +#[async_trait] +impl EventHandler for Handler { + async fn ready(&self, _: Context, ready: Ready) { + println!("{} is connected!", ready.user.name); + } +} + +#[group] +#[commands(deafen, join, leave, mute, play, ping, undeafen, unmute)] +struct General; + +#[tokio::main] +async fn main() { + tracing_subscriber::fmt::init(); + + // Configure the client with your Discord bot token in the environment. + let token = env::var("DISCORD_TOKEN").expect("Expected a token in the environment"); + + let mut framework = StandardFramework::new() + .group(&GENERAL_GROUP); + + framework.configure(|c| c.prefix("~")); + + let intents = GatewayIntents::non_privileged() | GatewayIntents::MESSAGE_CONTENT; + + let mut client = Client::builder(&token, intents) + .event_handler(Handler) + .framework(framework) + .register_songbird() + .await + .expect("Err creating client"); + + tokio::spawn(async move { + let _ = client + .start() + .await + .map_err(|why| println!("Client ended: {:?}", why)); + }); + + tokio::signal::ctrl_c().await; + println!("Received Ctrl-C, shutting down."); +} + +#[command] +#[only_in(guilds)] +async fn deafen(ctx: &Context, msg: &Message) -> CommandResult { + let guild = msg.guild(&ctx.cache).unwrap(); + let guild_id = guild.id; + + let manager = songbird::get(ctx) + .await + .expect("Songbird Voice client placed in at initialisation.") + .clone(); + + let handler_lock = match manager.get(guild_id) { + Some(handler) => handler, + None => { + check_msg(msg.reply(ctx, "Not in a voice channel").await); + + return Ok(()); + }, + }; + + let mut handler = handler_lock.lock().await; + + if handler.is_deaf() { + check_msg(msg.channel_id.say(&ctx.http, "Already deafened").await); + } else { + if let Err(e) = handler.deafen(true).await { + check_msg( + msg.channel_id + .say(&ctx.http, format!("Failed: {:?}", e)) + .await, + ); + } + + check_msg(msg.channel_id.say(&ctx.http, "Deafened").await); + } + + Ok(()) +} + +#[command] +#[only_in(guilds)] +async fn join(ctx: &Context, msg: &Message) -> CommandResult { + let guild = msg.guild(&ctx.cache).unwrap(); + let guild_id = guild.id; + + let channel_id = guild + .voice_states + .get(&msg.author.id) + .and_then(|voice_state| voice_state.channel_id); + + let connect_to = match channel_id { + Some(channel) => channel, + None => { + check_msg(msg.reply(ctx, "Not in a voice channel").await); + + return Ok(()); + }, + }; + + let manager = songbird::get(ctx) + .await + .expect("Songbird Voice client placed in at initialisation.") + .clone(); + + let (handler_lock, _success) = manager.join(guild_id, connect_to).await; + + // Attach an event handler to see notifications of all track errors. + let mut handler = handler_lock.lock().await; + handler.add_global_event(TrackEvent::Error.into(), TrackErrorNotifier); + + Ok(()) +} + +struct TrackErrorNotifier; + +#[async_trait] +impl VoiceEventHandler for TrackErrorNotifier { + async fn act(&self, ctx: &EventContext<'_>) -> Option { + if let EventContext::Track(track_list) = ctx { + for (state, handle) in *track_list { + println!( + "Track {:?} encountered an error: {:?}", + handle.uuid(), + state.playing + ); + } + } + + None + } +} + +#[command] +#[only_in(guilds)] +async fn leave(ctx: &Context, msg: &Message) -> CommandResult { + let guild = msg.guild(&ctx.cache).unwrap(); + let guild_id = guild.id; + + let manager = songbird::get(ctx) + .await + .expect("Songbird Voice client placed in at initialisation.") + .clone(); + let has_handler = manager.get(guild_id).is_some(); + + if has_handler { + if let Err(e) = manager.remove(guild_id).await { + check_msg( + msg.channel_id + .say(&ctx.http, format!("Failed: {:?}", e)) + .await, + ); + } + + check_msg(msg.channel_id.say(&ctx.http, "Left voice channel").await); + } else { + check_msg(msg.reply(ctx, "Not in a voice channel").await); + } + + Ok(()) +} + +#[command] +#[only_in(guilds)] +async fn mute(ctx: &Context, msg: &Message) -> CommandResult { + let guild = msg.guild(&ctx.cache).unwrap(); + let guild_id = guild.id; + + let manager = songbird::get(ctx) + .await + .expect("Songbird Voice client placed in at initialisation.") + .clone(); + + let handler_lock = match manager.get(guild_id) { + Some(handler) => handler, + None => { + check_msg(msg.reply(ctx, "Not in a voice channel").await); + + return Ok(()); + }, + }; + + let mut handler = handler_lock.lock().await; + + if handler.is_mute() { + check_msg(msg.channel_id.say(&ctx.http, "Already muted").await); + } else { + if let Err(e) = handler.mute(true).await { + check_msg( + msg.channel_id + .say(&ctx.http, format!("Failed: {:?}", e)) + .await, + ); + } + + check_msg(msg.channel_id.say(&ctx.http, "Now muted").await); + } + + Ok(()) +} + +#[command] +async fn ping(ctx: &Context, msg: &Message) -> CommandResult { + check_msg(msg.channel_id.say(&ctx.http, "Pong!").await); + Ok(()) +} + +#[command] +#[only_in(guilds)] +async fn play(ctx: &Context, msg: &Message, mut args: Args) -> CommandResult { + let url = match args.single::() { + Ok(url) => url, + Err(_) => { + check_msg( + msg.channel_id + .say(&ctx.http, "Must provide a URL to a video or audio") + .await, + ); + + return Ok(()); + }, + }; + + let time = match args.single::() { + Ok(t) => t, + Err(_) => { + check_msg( + msg.channel_id + .say(&ctx.http, "Must provide a valid f64 timestamp to jump to.") + .await, + ); + + return Ok(()); + }, + }; + + if !url.starts_with("http") { + check_msg( + msg.channel_id + .say(&ctx.http, "Must provide a valid URL") + .await, + ); + + return Ok(()); + } + + let guild = msg.guild(&ctx.cache).unwrap(); + let guild_id = guild.id; + + let manager = songbird::get(ctx) + .await + .expect("Songbird Voice client placed in at initialisation.") + .clone(); + + if let Some(handler_lock) = manager.get(guild_id) { + let mut handler = handler_lock.lock().await; + + let jar = reqwest::cookie::Jar::default(); + jar.add_cookie_str("CONSENT=YES+; Path=/; Domain=youtube.com; Secure; Expires=Fri, 01 Jan 2038 00:00:00 GMT;", &"https://youtube.com".parse().unwrap()); + + let src = songbird::input::YoutubeDl::new_ytdl_like("yt-dlp", reqwest::Client::builder().cookie_provider(std::sync::Arc::new(jar)).build().unwrap(), url); + + let h = handler.play_input(src.into()); + + h.add_event( + Event::Delayed(Duration::from_secs(10)), + SkipHandler { + skip_to: Duration::from_secs_f64(time), + }, + ); + + check_msg(msg.channel_id.say(&ctx.http, "Playing song").await); + } else { + check_msg( + msg.channel_id + .say(&ctx.http, "Not in a voice channel to play in") + .await, + ); + } + + Ok(()) +} + +struct SkipHandler { + skip_to: Duration, +} + +#[async_trait] +impl VoiceEventHandler for SkipHandler { + async fn act(&self, ctx: &EventContext<'_>) -> Option { + if let EventContext::Track(&[(state, handle)]) = ctx { + println!("Current position is {:?}", state.position); + println!("Seeking to {:?}", self.skip_to); + let resp = handle.seek_time(self.skip_to); + println!("Seek response {:?}", resp); + return None; + } + + println!("This wasn't supposed to happen."); + None + } +} + +#[command] +#[only_in(guilds)] +async fn undeafen(ctx: &Context, msg: &Message) -> CommandResult { + let guild = msg.guild(&ctx.cache).unwrap(); + let guild_id = guild.id; + + let manager = songbird::get(ctx) + .await + .expect("Songbird Voice client placed in at initialisation.") + .clone(); + + if let Some(handler_lock) = manager.get(guild_id) { + let mut handler = handler_lock.lock().await; + if let Err(e) = handler.deafen(false).await { + check_msg( + msg.channel_id + .say(&ctx.http, format!("Failed: {:?}", e)) + .await, + ); + } + + check_msg(msg.channel_id.say(&ctx.http, "Undeafened").await); + } else { + check_msg( + msg.channel_id + .say(&ctx.http, "Not in a voice channel to undeafen in") + .await, + ); + } + + Ok(()) +} + +#[command] +#[only_in(guilds)] +async fn unmute(ctx: &Context, msg: &Message) -> CommandResult { + let guild = msg.guild(&ctx.cache).unwrap(); + let guild_id = guild.id; + + let manager = songbird::get(ctx) + .await + .expect("Songbird Voice client placed in at initialisation.") + .clone(); + + if let Some(handler_lock) = manager.get(guild_id) { + let mut handler = handler_lock.lock().await; + if let Err(e) = handler.mute(false).await { + check_msg( + msg.channel_id + .say(&ctx.http, format!("Failed: {:?}", e)) + .await, + ); + } + + check_msg(msg.channel_id.say(&ctx.http, "Unmuted").await); + } else { + check_msg( + msg.channel_id + .say(&ctx.http, "Not in a voice channel to unmute in") + .await, + ); + } + + Ok(()) +} + +/// Checks that a message successfully sent; if not, then logs why to stdout. +fn check_msg(result: SerenityResult) { + if let Err(why) = result { + println!("Error sending message: {:?}", why); + } +} diff --git a/examples/serenity/voice/src/main.rs b/examples/serenity/voice/src/main.rs index 8afcd25..7157cf0 100644 --- a/examples/serenity/voice/src/main.rs +++ b/examples/serenity/voice/src/main.rs @@ -13,6 +13,15 @@ use std::env; // The voice client can be retrieved in any command using `songbird::get(ctx).await`. use songbird::SerenityInit; +// Event related imports to detect track creation failures. +use songbird::events::{Event, EventContext, EventHandler as VoiceEventHandler, TrackEvent}; + +// To turn user URLs into playable audio, we'll use yt-dlp. +use songbird::input::YoutubeDl; + +// YtDl requests need an HTTP client to operate -- we'll create and store our own. +use reqwest::Client as HttpClient; + // Import the `Context` to handle commands. use serenity::client::Context; @@ -20,17 +29,24 @@ use serenity::{ async_trait, client::{Client, EventHandler}, framework::{ - StandardFramework, standard::{ - Args, CommandResult, macros::{command, group}, + Args, + CommandResult, }, + StandardFramework, }, model::{channel::Message, gateway::Ready}, - prelude::GatewayIntents, + prelude::{GatewayIntents, TypeMapKey}, Result as SerenityResult, }; +struct HttpKey; + +impl TypeMapKey for HttpKey { + type Value = HttpClient; +} + struct Handler; #[async_trait] @@ -47,42 +63,49 @@ struct General; #[tokio::main] async fn main() { tracing_subscriber::fmt::init(); - + // Configure the client with your Discord bot token in the environment. - let token = env::var("DISCORD_TOKEN") - .expect("Expected a token in the environment"); + let token = env::var("DISCORD_TOKEN").expect("Expected a token in the environment"); let framework = StandardFramework::new() - .configure(|c| c - .prefix("~")) .group(&GENERAL_GROUP); + framework.configure(|c| c.prefix("~")); - let intents = GatewayIntents::non_privileged() - | GatewayIntents::MESSAGE_CONTENT; + let intents = GatewayIntents::non_privileged() | GatewayIntents::MESSAGE_CONTENT; let mut client = Client::builder(&token, intents) .event_handler(Handler) .framework(framework) .register_songbird() + // We insert our own HTTP client here to make use of in + // `~play`. If we wanted, we could supply cookies and auth + // details ahead of time. + // + // Generally, we don't want to make a new Client for every request! + .type_map_insert::(HttpClient::new()) .await .expect("Err creating client"); tokio::spawn(async move { - let _ = client.start().await.map_err(|why| println!("Client ended: {:?}", why)); + let _ = client + .start() + .await + .map_err(|why| println!("Client ended: {:?}", why)); }); - - tokio::signal::ctrl_c().await; + + let _signal_err = tokio::signal::ctrl_c().await; println!("Received Ctrl-C, shutting down."); } #[command] #[only_in(guilds)] async fn deafen(ctx: &Context, msg: &Message) -> CommandResult { - let guild = msg.guild(&ctx.cache).unwrap(); - let guild_id = guild.id; + let guild_id = msg.guild_id.unwrap(); - let manager = songbird::get(ctx).await - .expect("Songbird Voice client placed in at initialisation.").clone(); + let manager = songbird::get(ctx) + .await + .expect("Songbird Voice client placed in at initialisation.") + .clone(); let handler_lock = match manager.get(guild_id) { Some(handler) => handler, @@ -99,7 +122,11 @@ async fn deafen(ctx: &Context, msg: &Message) -> CommandResult { check_msg(msg.channel_id.say(&ctx.http, "Already deafened").await); } else { if let Err(e) = handler.deafen(true).await { - check_msg(msg.channel_id.say(&ctx.http, format!("Failed: {:?}", e)).await); + check_msg( + msg.channel_id + .say(&ctx.http, format!("Failed: {:?}", e)) + .await, + ); } check_msg(msg.channel_id.say(&ctx.http, "Deafened").await); @@ -111,12 +138,15 @@ async fn deafen(ctx: &Context, msg: &Message) -> CommandResult { #[command] #[only_in(guilds)] async fn join(ctx: &Context, msg: &Message) -> CommandResult { - let guild = msg.guild(&ctx.cache).unwrap(); - let guild_id = guild.id; + let (guild_id, channel_id) = { + let guild = msg.guild(&ctx.cache).unwrap(); + let channel_id = guild + .voice_states + .get(&msg.author.id) + .and_then(|voice_state| voice_state.channel_id); - let channel_id = guild - .voice_states.get(&msg.author.id) - .and_then(|voice_state| voice_state.channel_id); + (guild.id, channel_id) + }; let connect_to = match channel_id { Some(channel) => channel, @@ -124,30 +154,60 @@ async fn join(ctx: &Context, msg: &Message) -> CommandResult { check_msg(msg.reply(ctx, "Not in a voice channel").await); return Ok(()); - } + }, }; - let manager = songbird::get(ctx).await - .expect("Songbird Voice client placed in at initialisation.").clone(); + let manager = songbird::get(ctx) + .await + .expect("Songbird Voice client placed in at initialisation.") + .clone(); - let _handler = manager.join(guild_id, connect_to).await; + let (handler_lock, _success) = manager.join(guild_id, connect_to).await; + + // Attach an event handler to see notifications of all track errors. + let mut handler = handler_lock.lock().await; + handler.add_global_event(TrackEvent::Error.into(), TrackErrorNotifier); Ok(()) } +struct TrackErrorNotifier; + +#[async_trait] +impl VoiceEventHandler for TrackErrorNotifier { + async fn act(&self, ctx: &EventContext<'_>) -> Option { + if let EventContext::Track(track_list) = ctx { + for (state, handle) in *track_list { + println!( + "Track {:?} encountered an error: {:?}", + handle.uuid(), + state.playing + ); + } + } + + None + } +} + #[command] #[only_in(guilds)] async fn leave(ctx: &Context, msg: &Message) -> CommandResult { - let guild = msg.guild(&ctx.cache).unwrap(); - let guild_id = guild.id; + let guild_id = msg.guild_id.unwrap(); - let manager = songbird::get(ctx).await - .expect("Songbird Voice client placed in at initialisation.").clone(); + let manager = songbird::get(ctx) + .await + .expect("Songbird Voice client placed in at initialisation.") + .clone(); let has_handler = manager.get(guild_id).is_some(); if has_handler { if let Err(e) = manager.remove(guild_id).await { - check_msg(msg.channel_id.say(&ctx.http, format!("Failed: {:?}", e)).await); + check_msg( + msg.channel_id + .say(&ctx.http, format!("Failed: {:?}", e)) + .await, + ); } check_msg(msg.channel_id.say(&ctx.http, "Left voice channel").await); @@ -161,11 +221,12 @@ async fn leave(ctx: &Context, msg: &Message) -> CommandResult { #[command] #[only_in(guilds)] async fn mute(ctx: &Context, msg: &Message) -> CommandResult { - let guild = msg.guild(&ctx.cache).unwrap(); - let guild_id = guild.id; + let guild_id = msg.guild_id.unwrap(); - let manager = songbird::get(ctx).await - .expect("Songbird Voice client placed in at initialisation.").clone(); + let manager = songbird::get(ctx) + .await + .expect("Songbird Voice client placed in at initialisation.") + .clone(); let handler_lock = match manager.get(guild_id) { Some(handler) => handler, @@ -182,7 +243,11 @@ async fn mute(ctx: &Context, msg: &Message) -> CommandResult { check_msg(msg.channel_id.say(&ctx.http, "Already muted").await); } else { if let Err(e) = handler.mute(true).await { - check_msg(msg.channel_id.say(&ctx.http, format!("Failed: {:?}", e)).await); + check_msg( + msg.channel_id + .say(&ctx.http, format!("Failed: {:?}", e)) + .await, + ); } check_msg(msg.channel_id.say(&ctx.http, "Now muted").await); @@ -192,9 +257,8 @@ async fn mute(ctx: &Context, msg: &Message) -> CommandResult { } #[command] -async fn ping(context: &Context, msg: &Message) -> CommandResult { - check_msg(msg.channel_id.say(&context.http, "Pong!").await); - +async fn ping(ctx: &Context, msg: &Message) -> CommandResult { + check_msg(msg.channel_id.say(&ctx.http, "Pong!").await); Ok(()) } @@ -204,43 +268,53 @@ async fn play(ctx: &Context, msg: &Message, mut args: Args) -> CommandResult { let url = match args.single::() { Ok(url) => url, Err(_) => { - check_msg(msg.channel_id.say(&ctx.http, "Must provide a URL to a video or audio").await); + check_msg( + msg.channel_id + .say(&ctx.http, "Must provide a URL to a video or audio") + .await, + ); return Ok(()); }, }; if !url.starts_with("http") { - check_msg(msg.channel_id.say(&ctx.http, "Must provide a valid URL").await); + check_msg( + msg.channel_id + .say(&ctx.http, "Must provide a valid URL") + .await, + ); return Ok(()); } - let guild = msg.guild(&ctx.cache).unwrap(); - let guild_id = guild.id; + let guild_id = msg.guild_id.unwrap(); - let manager = songbird::get(ctx).await - .expect("Songbird Voice client placed in at initialisation.").clone(); + let http_client = { + let data = ctx.data.read().await; + data.get::() + .cloned() + .expect("Guaranteed to exist in the typemap.") + }; + + let manager = songbird::get(ctx) + .await + .expect("Songbird Voice client placed in at initialisation.") + .clone(); if let Some(handler_lock) = manager.get(guild_id) { let mut handler = handler_lock.lock().await; - let source = match songbird::ytdl(&url).await { - Ok(source) => source, - Err(why) => { - println!("Err starting source: {:?}", why); - - check_msg(msg.channel_id.say(&ctx.http, "Error sourcing ffmpeg").await); - - return Ok(()); - }, - }; - - handler.play_source(source); + let src = YoutubeDl::new(http_client, url); + let _ = handler.play_input(src.into()); check_msg(msg.channel_id.say(&ctx.http, "Playing song").await); } else { - check_msg(msg.channel_id.say(&ctx.http, "Not in a voice channel to play in").await); + check_msg( + msg.channel_id + .say(&ctx.http, "Not in a voice channel to play in") + .await, + ); } Ok(()) @@ -249,21 +323,30 @@ async fn play(ctx: &Context, msg: &Message, mut args: Args) -> CommandResult { #[command] #[only_in(guilds)] async fn undeafen(ctx: &Context, msg: &Message) -> CommandResult { - let guild = msg.guild(&ctx.cache).unwrap(); - let guild_id = guild.id; + let guild_id = msg.guild_id.unwrap(); - let manager = songbird::get(ctx).await - .expect("Songbird Voice client placed in at initialisation.").clone(); + let manager = songbird::get(ctx) + .await + .expect("Songbird Voice client placed in at initialisation.") + .clone(); if let Some(handler_lock) = manager.get(guild_id) { let mut handler = handler_lock.lock().await; if let Err(e) = handler.deafen(false).await { - check_msg(msg.channel_id.say(&ctx.http, format!("Failed: {:?}", e)).await); + check_msg( + msg.channel_id + .say(&ctx.http, format!("Failed: {:?}", e)) + .await, + ); } check_msg(msg.channel_id.say(&ctx.http, "Undeafened").await); } else { - check_msg(msg.channel_id.say(&ctx.http, "Not in a voice channel to undeafen in").await); + check_msg( + msg.channel_id + .say(&ctx.http, "Not in a voice channel to undeafen in") + .await, + ); } Ok(()) @@ -272,21 +355,30 @@ async fn undeafen(ctx: &Context, msg: &Message) -> CommandResult { #[command] #[only_in(guilds)] async fn unmute(ctx: &Context, msg: &Message) -> CommandResult { - let guild = msg.guild(&ctx.cache).unwrap(); - let guild_id = guild.id; - - let manager = songbird::get(ctx).await - .expect("Songbird Voice client placed in at initialisation.").clone(); + let guild_id = msg.guild_id.unwrap(); + + let manager = songbird::get(ctx) + .await + .expect("Songbird Voice client placed in at initialisation.") + .clone(); if let Some(handler_lock) = manager.get(guild_id) { let mut handler = handler_lock.lock().await; if let Err(e) = handler.mute(false).await { - check_msg(msg.channel_id.say(&ctx.http, format!("Failed: {:?}", e)).await); + check_msg( + msg.channel_id + .say(&ctx.http, format!("Failed: {:?}", e)) + .await, + ); } check_msg(msg.channel_id.say(&ctx.http, "Unmuted").await); } else { - check_msg(msg.channel_id.say(&ctx.http, "Not in a voice channel to unmute in").await); + check_msg( + msg.channel_id + .say(&ctx.http, "Not in a voice channel to unmute in") + .await, + ); } Ok(()) diff --git a/examples/serenity/voice_cached_audio/.gitignore b/examples/serenity/voice_cached_audio/.gitignore new file mode 100644 index 0000000..e9d1220 --- /dev/null +++ b/examples/serenity/voice_cached_audio/.gitignore @@ -0,0 +1 @@ +*.dca diff --git a/examples/serenity/voice_storage/Cargo.toml b/examples/serenity/voice_cached_audio/Cargo.toml similarity index 69% rename from examples/serenity/voice_storage/Cargo.toml rename to examples/serenity/voice_cached_audio/Cargo.toml index b13b25c..729e10c 100644 --- a/examples/serenity/voice_storage/Cargo.toml +++ b/examples/serenity/voice_cached_audio/Cargo.toml @@ -1,8 +1,8 @@ [package] -name = "voice_storage" +name = "voice_cached_audio" version = "0.1.0" authors = ["my name "] -edition = "2018" +edition = "2021" [dependencies] tracing = "0.1" @@ -16,6 +16,12 @@ path = "../../../" version = "0.11" features = ["cache", "framework", "standard_framework", "voice", "http", "rustls_backend"] +[dependencies.symphonia] +version = "0.5" +features = ["mp3"] +git = "https://github.com/FelixMcFelix/Symphonia" +branch = "songbird-fixes" + [dependencies.tokio] version = "1.0" features = ["macros", "rt-multi-thread"] diff --git a/examples/serenity/voice_storage/src/main.rs b/examples/serenity/voice_cached_audio/src/main.rs similarity index 62% rename from examples/serenity/voice_storage/src/main.rs rename to examples/serenity/voice_cached_audio/src/main.rs index f5c9c53..a7b3a82 100644 --- a/examples/serenity/voice_storage/src/main.rs +++ b/examples/serenity/voice_cached_audio/src/main.rs @@ -9,17 +9,23 @@ //! git = "https://github.com/serenity-rs/serenity.git" //! features = ["cache", "framework", "standard_framework", "voice"] //! ``` -use std::{collections::HashMap, convert::TryInto, env, sync::{Arc, Weak}}; +use std::{ + collections::HashMap, + convert::TryInto, + env, + sync::{Arc, Weak}, +}; use serenity::{ async_trait, client::{Client, Context, EventHandler}, framework::{ - StandardFramework, standard::{ - Args, CommandResult, macros::{command, group}, + Args, + CommandResult, }, + StandardFramework, }, model::{channel::Message, gateway::Ready}, prelude::{GatewayIntents, Mentionable, Mutex}, @@ -29,8 +35,8 @@ use serenity::{ use songbird::{ driver::Bitrate, input::{ - self, cached::{Compressed, Memory}, + File, Input, }, Call, @@ -62,9 +68,9 @@ impl From<&CachedSound> for Input { fn from(obj: &CachedSound) -> Self { use CachedSound::*; match obj { - Compressed(c) => c.new_handle() - .into(), - Uncompressed(u) => u.new_handle() + Compressed(c) => c.new_handle().into(), + Uncompressed(u) => u + .new_handle() .try_into() .expect("Failed to create decoder for Memory source."), } @@ -86,16 +92,13 @@ async fn main() { tracing_subscriber::fmt::init(); // Configure the client with your Discord bot token in the environment. - let token = env::var("DISCORD_TOKEN") - .expect("Expected a token in the environment"); + let token = env::var("DISCORD_TOKEN").expect("Expected a token in the environment"); let framework = StandardFramework::new() - .configure(|c| c - .prefix("~")) .group(&GENERAL_GROUP); + framework.configure(|c| c.prefix("~")); - let intents = GatewayIntents::non_privileged() - | GatewayIntents::MESSAGE_CONTENT; + let intents = GatewayIntents::non_privileged() | GatewayIntents::MESSAGE_CONTENT; let mut client = Client::builder(&token, intents) .event_handler(Handler) @@ -117,19 +120,19 @@ async fn main() { // // This is a small sound effect, so storing the whole thing is relatively cheap. // - // `spawn_loader` creates a new thread which works to copy all the audio into memory + // `spawn_loader` creates a new thread which works to copy all the audio into memory // ahead of time. We do this in both cases to ensure optimal performance for the audio // core. - let ting_src = Memory::new( - input::ffmpeg("ting.wav").await.expect("File should be in root folder."), - ).expect("These parameters are well-defined."); + let ting_src = Memory::new(File::new("../../../resources/ting.wav").into()) + .await + .expect("These parameters are well-defined."); let _ = ting_src.raw.spawn_loader(); audio_map.insert("ting".into(), CachedSound::Uncompressed(ting_src)); // Another short sting, to show where each loop occurs. - let loop_src = Memory::new( - input::ffmpeg("loop.wav").await.expect("File should be in root folder."), - ).expect("These parameters are well-defined."); + let loop_src = Memory::new(File::new("../../../resources/loop.wav").into()) + .await + .expect("These parameters are well-defined."); let _ = loop_src.raw.spawn_loader(); audio_map.insert("loop".into(), CachedSound::Uncompressed(loop_src)); @@ -139,26 +142,42 @@ async fn main() { // // Music by Cloudkicker, used under CC BY-SC-SA 3.0 (https://creativecommons.org/licenses/by-nc-sa/3.0/). let song_src = Compressed::new( - input::ffmpeg("Cloudkicker_-_Loops_-_22_2011_07.mp3").await.expect("Link may be dead."), - Bitrate::BitsPerSecond(128_000), - ).expect("These parameters are well-defined."); + File::new("../../../resources/Cloudkicker - 2011 07.mp3").into(), + Bitrate::BitsPerSecond(128_000), + ) + .await + .expect("These parameters are well-defined."); let _ = song_src.raw.spawn_loader(); + + // Compressed sources are internally stored as DCA1 format files. + // Because `Compressed` implements `std::io::Read`, we can save these + // to disk and use them again later if we want! + let mut creator = song_src.new_handle(); + std::thread::spawn(move || { + let mut out_file = std::fs::File::create("ckick-dca1.dca").unwrap(); + std::io::copy(&mut creator, &mut out_file).expect("Error writing out song!"); + }); + audio_map.insert("song".into(), CachedSound::Compressed(song_src)); data.insert::(Arc::new(Mutex::new(audio_map))); } - let _ = client.start().await.map_err(|why| println!("Client ended: {:?}", why)); + let _ = client + .start() + .await + .map_err(|why| println!("Client ended: {:?}", why)); } #[command] #[only_in(guilds)] async fn deafen(ctx: &Context, msg: &Message) -> CommandResult { - let guild = msg.guild(&ctx.cache).unwrap(); - let guild_id = guild.id; + let guild_id = msg.guild_id.unwrap(); - let manager = songbird::get(ctx).await - .expect("Songbird Voice client placed in at initialisation.").clone(); + let manager = songbird::get(ctx) + .await + .expect("Songbird Voice client placed in at initialisation.") + .clone(); let handler_lock = match manager.get(guild_id) { Some(handler) => handler, @@ -175,7 +194,11 @@ async fn deafen(ctx: &Context, msg: &Message) -> CommandResult { check_msg(msg.channel_id.say(&ctx.http, "Already deafened").await); } else { if let Err(e) = handler.deafen(true).await { - check_msg(msg.channel_id.say(&ctx.http, format!("Failed: {:?}", e)).await); + check_msg( + msg.channel_id + .say(&ctx.http, format!("Failed: {:?}", e)) + .await, + ); } check_msg(msg.channel_id.say(&ctx.http, "Deafened").await); @@ -187,13 +210,15 @@ async fn deafen(ctx: &Context, msg: &Message) -> CommandResult { #[command] #[only_in(guilds)] async fn join(ctx: &Context, msg: &Message) -> CommandResult { - let guild = msg.guild(&ctx.cache).unwrap(); - let guild_id = guild.id; - - let channel_id = guild - .voice_states.get(&msg.author.id) - .and_then(|voice_state| voice_state.channel_id); + let (guild_id, channel_id) = { + let guild = msg.guild(&ctx.cache).unwrap(); + let channel_id = guild + .voice_states + .get(&msg.author.id) + .and_then(|voice_state| voice_state.channel_id); + (guild.id, channel_id) + }; let connect_to = match channel_id { Some(channel) => channel, @@ -201,11 +226,13 @@ async fn join(ctx: &Context, msg: &Message) -> CommandResult { check_msg(msg.reply(ctx, "Not in a voice channel").await); return Ok(()); - } + }, }; - let manager = songbird::get(ctx).await - .expect("Songbird Voice client placed in at initialisation.").clone(); + let manager = songbird::get(ctx) + .await + .expect("Songbird Voice client placed in at initialisation.") + .clone(); let (handler_lock, success_reader) = manager.join(guild_id, connect_to).await; @@ -213,14 +240,26 @@ async fn join(ctx: &Context, msg: &Message) -> CommandResult { if let Ok(_reader) = success_reader { let mut handler = handler_lock.lock().await; - check_msg(msg.channel_id.say(&ctx.http, &format!("Joined {}", connect_to.mention())).await); + check_msg( + msg.channel_id + .say(&ctx.http, &format!("Joined {}", connect_to.mention())) + .await, + ); - let sources_lock = ctx.data.read().await.get::().cloned().expect("Sound cache was installed at startup."); + let sources_lock = ctx + .data + .read() + .await + .get::() + .cloned() + .expect("Sound cache was installed at startup."); let sources_lock_for_evt = sources_lock.clone(); let sources = sources_lock.lock().await; - let source = sources.get("song").expect("Handle placed into cache at startup."); + let source = sources + .get("song") + .expect("Handle placed into cache at startup."); - let song = handler.play_source(source.into()); + let song = handler.play_input(source.into()); let _ = song.set_volume(1.0); let _ = song.enable_loop(); @@ -233,7 +272,11 @@ async fn join(ctx: &Context, msg: &Message) -> CommandResult { }, ); } else { - check_msg(msg.channel_id.say(&ctx.http, "Error joining the channel").await); + check_msg( + msg.channel_id + .say(&ctx.http, "Error joining the channel") + .await, + ); } Ok(()) @@ -250,11 +293,14 @@ impl VoiceEventHandler for LoopPlaySound { if let Some(call_lock) = self.call_lock.upgrade() { let src = { let sources = self.sources.lock().await; - sources.get("loop").expect("Handle placed into cache at startup.").into() + sources + .get("loop") + .expect("Handle placed into cache at startup.") + .into() }; let mut handler = call_lock.lock().await; - let sound = handler.play_source(src); + let sound = handler.play_input(src); let _ = sound.set_volume(0.5); } @@ -265,16 +311,21 @@ impl VoiceEventHandler for LoopPlaySound { #[command] #[only_in(guilds)] async fn leave(ctx: &Context, msg: &Message) -> CommandResult { - let guild = msg.guild(&ctx.cache).unwrap(); - let guild_id = guild.id; + let guild_id = msg.guild_id.unwrap(); - let manager = songbird::get(ctx).await - .expect("Songbird Voice client placed in at initialisation.").clone(); + let manager = songbird::get(ctx) + .await + .expect("Songbird Voice client placed in at initialisation.") + .clone(); let has_handler = manager.get(guild_id).is_some(); if has_handler { if let Err(e) = manager.remove(guild_id).await { - check_msg(msg.channel_id.say(&ctx.http, format!("Failed: {:?}", e)).await); + check_msg( + msg.channel_id + .say(&ctx.http, format!("Failed: {:?}", e)) + .await, + ); } check_msg(msg.channel_id.say(&ctx.http, "Left voice channel").await); @@ -288,11 +339,12 @@ async fn leave(ctx: &Context, msg: &Message) -> CommandResult { #[command] #[only_in(guilds)] async fn mute(ctx: &Context, msg: &Message) -> CommandResult { - let guild = msg.guild(&ctx.cache).unwrap(); - let guild_id = guild.id; + let guild_id = msg.guild_id.unwrap(); - let manager = songbird::get(ctx).await - .expect("Songbird Voice client placed in at initialisation.").clone(); + let manager = songbird::get(ctx) + .await + .expect("Songbird Voice client placed in at initialisation.") + .clone(); let handler_lock = match manager.get(guild_id) { Some(handler) => handler, @@ -309,7 +361,11 @@ async fn mute(ctx: &Context, msg: &Message) -> CommandResult { check_msg(msg.channel_id.say(&ctx.http, "Already muted").await); } else { if let Err(e) = handler.mute(true).await { - check_msg(msg.channel_id.say(&ctx.http, format!("Failed: {:?}", e)).await); + check_msg( + msg.channel_id + .say(&ctx.http, format!("Failed: {:?}", e)) + .await, + ); } check_msg(msg.channel_id.say(&ctx.http, "Now muted").await); @@ -321,24 +377,37 @@ async fn mute(ctx: &Context, msg: &Message) -> CommandResult { #[command] #[only_in(guilds)] async fn ting(ctx: &Context, msg: &Message, _args: Args) -> CommandResult { - let guild = msg.guild(&ctx.cache).unwrap(); - let guild_id = guild.id; + let guild_id = msg.guild_id.unwrap(); - let manager = songbird::get(ctx).await - .expect("Songbird Voice client placed in at initialisation.").clone(); + let manager = songbird::get(ctx) + .await + .expect("Songbird Voice client placed in at initialisation.") + .clone(); if let Some(handler_lock) = manager.get(guild_id) { let mut handler = handler_lock.lock().await; - let sources_lock = ctx.data.read().await.get::().cloned().expect("Sound cache was installed at startup."); + let sources_lock = ctx + .data + .read() + .await + .get::() + .cloned() + .expect("Sound cache was installed at startup."); let sources = sources_lock.lock().await; - let source = sources.get("ting").expect("Handle placed into cache at startup."); + let source = sources + .get("ting") + .expect("Handle placed into cache at startup."); - let _sound = handler.play_source(source.into()); + let _sound = handler.play_input(source.into()); check_msg(msg.channel_id.say(&ctx.http, "Ting!").await); } else { - check_msg(msg.channel_id.say(&ctx.http, "Not in a voice channel to play in").await); + check_msg( + msg.channel_id + .say(&ctx.http, "Not in a voice channel to play in") + .await, + ); } Ok(()) @@ -347,22 +416,31 @@ async fn ting(ctx: &Context, msg: &Message, _args: Args) -> CommandResult { #[command] #[only_in(guilds)] async fn undeafen(ctx: &Context, msg: &Message) -> CommandResult { - let guild = msg.guild(&ctx.cache).unwrap(); - let guild_id = guild.id; + let guild_id = msg.guild_id.unwrap(); - let manager = songbird::get(ctx).await - .expect("Songbird Voice client placed in at initialisation.").clone(); + let manager = songbird::get(ctx) + .await + .expect("Songbird Voice client placed in at initialisation.") + .clone(); if let Some(handler_lock) = manager.get(guild_id) { let mut handler = handler_lock.lock().await; if let Err(e) = handler.deafen(false).await { - check_msg(msg.channel_id.say(&ctx.http, format!("Failed: {:?}", e)).await); + check_msg( + msg.channel_id + .say(&ctx.http, format!("Failed: {:?}", e)) + .await, + ); } check_msg(msg.channel_id.say(&ctx.http, "Undeafened").await); } else { - check_msg(msg.channel_id.say(&ctx.http, "Not in a voice channel to undeafen in").await); + check_msg( + msg.channel_id + .say(&ctx.http, "Not in a voice channel to undeafen in") + .await, + ); } Ok(()) @@ -371,21 +449,30 @@ async fn undeafen(ctx: &Context, msg: &Message) -> CommandResult { #[command] #[only_in(guilds)] async fn unmute(ctx: &Context, msg: &Message) -> CommandResult { - let guild = msg.guild(&ctx.cache).unwrap(); - let guild_id = guild.id; - let manager = songbird::get(ctx).await - .expect("Songbird Voice client placed in at initialisation.").clone(); + let guild_id = msg.guild_id.unwrap(); + let manager = songbird::get(ctx) + .await + .expect("Songbird Voice client placed in at initialisation.") + .clone(); if let Some(handler_lock) = manager.get(guild_id) { let mut handler = handler_lock.lock().await; if let Err(e) = handler.mute(false).await { - check_msg(msg.channel_id.say(&ctx.http, format!("Failed: {:?}", e)).await); + check_msg( + msg.channel_id + .say(&ctx.http, format!("Failed: {:?}", e)) + .await, + ); } check_msg(msg.channel_id.say(&ctx.http, "Unmuted").await); } else { - check_msg(msg.channel_id.say(&ctx.http, "Not in a voice channel to unmute in").await); + check_msg( + msg.channel_id + .say(&ctx.http, "Not in a voice channel to unmute in") + .await, + ); } Ok(()) diff --git a/examples/serenity/voice_events_queue/Cargo.toml b/examples/serenity/voice_events_queue/Cargo.toml index f47e8da..2d5f258 100644 --- a/examples/serenity/voice_events_queue/Cargo.toml +++ b/examples/serenity/voice_events_queue/Cargo.toml @@ -2,9 +2,10 @@ name = "voice_events_queue" version = "0.1.0" authors = ["my name "] -edition = "2018" +edition = "2021" [dependencies] +reqwest = "0.11" tracing = "0.1" tracing-subscriber = "0.2" tracing-futures = "0.2" @@ -17,6 +18,12 @@ path = "../../../" version = "0.11" features = ["cache", "standard_framework", "voice", "rustls_backend"] +[dependencies.symphonia] +version = "0.5" +features = ["aac", "mp3", "isomp4", "alac"] +git = "https://github.com/FelixMcFelix/Symphonia" +branch = "songbird-fixes" + [dependencies.tokio] version = "1.0" -features = ["macros", "rt-multi-thread"] +features = ["macros", "rt-multi-thread", "signal"] diff --git a/examples/serenity/voice_events_queue/src/main.rs b/examples/serenity/voice_events_queue/src/main.rs index 7a78d0d..0300d5e 100644 --- a/examples/serenity/voice_events_queue/src/main.rs +++ b/examples/serenity/voice_events_queue/src/main.rs @@ -18,6 +18,8 @@ use std::{ time::Duration, }; +use reqwest::Client as HttpClient; + use serenity::{ async_trait, client::{Client, Context, EventHandler}, @@ -31,15 +33,12 @@ use serenity::{ }, http::Http, model::{channel::Message, gateway::Ready, prelude::ChannelId}, - prelude::{GatewayIntents, Mentionable}, + prelude::{GatewayIntents, Mentionable, TypeMapKey}, Result as SerenityResult, }; use songbird::{ - input::{ - self, - restartable::Restartable, - }, + input::YoutubeDl, Event, EventContext, EventHandler as VoiceEventHandler, @@ -47,6 +46,12 @@ use songbird::{ TrackEvent, }; +struct HttpKey; + +impl TypeMapKey for HttpKey { + type Value = HttpClient; +} + struct Handler; #[async_trait] @@ -70,16 +75,16 @@ async fn main() { let token = env::var("DISCORD_TOKEN").expect("Expected a token in the environment"); let framework = StandardFramework::new() - .configure(|c| c.prefix("~")) .group(&GENERAL_GROUP); + framework.configure(|c| c.prefix("~")); - let intents = GatewayIntents::non_privileged() - | GatewayIntents::MESSAGE_CONTENT; + let intents = GatewayIntents::non_privileged() | GatewayIntents::MESSAGE_CONTENT; let mut client = Client::builder(&token, intents) .event_handler(Handler) .framework(framework) .register_songbird() + .type_map_insert::(HttpClient::new()) .await .expect("Err creating client"); @@ -87,12 +92,28 @@ async fn main() { .start() .await .map_err(|why| println!("Client ended: {:?}", why)); + + tokio::spawn(async move { + let _ = client + .start() + .await + .map_err(|why| println!("Client ended: {:?}", why)); + }); + + let _signal_err = tokio::signal::ctrl_c().await; + println!("Received Ctrl-C, shutting down."); +} + +async fn get_http_client(ctx: &Context) -> HttpClient { + let data = ctx.data.read().await; + data.get::() + .cloned() + .expect("Guaranteed to exist in the typemap.") } #[command] async fn deafen(ctx: &Context, msg: &Message) -> CommandResult { - let guild = msg.guild(&ctx.cache).unwrap(); - let guild_id = guild.id; + let guild_id = msg.guild(&ctx.cache).unwrap().id; let manager = songbird::get(ctx) .await @@ -130,13 +151,15 @@ async fn deafen(ctx: &Context, msg: &Message) -> CommandResult { #[command] #[only_in(guilds)] async fn join(ctx: &Context, msg: &Message) -> CommandResult { - let guild = msg.guild(&ctx.cache).unwrap(); - let guild_id = guild.id; + let (guild_id, channel_id) = { + let guild = msg.guild(&ctx.cache).unwrap(); + let channel_id = guild + .voice_states + .get(&msg.author.id) + .and_then(|voice_state| voice_state.channel_id); - let channel_id = guild - .voice_states - .get(&msg.author.id) - .and_then(|voice_state| voice_state.channel_id); + (guild.id, channel_id) + }; let connect_to = match channel_id { Some(channel) => channel, @@ -245,8 +268,7 @@ impl VoiceEventHandler for ChannelDurationNotifier { #[command] #[only_in(guilds)] async fn leave(ctx: &Context, msg: &Message) -> CommandResult { - let guild = msg.guild(&ctx.cache).unwrap(); - let guild_id = guild.id; + let guild_id = msg.guild(&ctx.cache).unwrap().id; let manager = songbird::get(ctx) .await @@ -274,8 +296,7 @@ async fn leave(ctx: &Context, msg: &Message) -> CommandResult { #[command] #[only_in(guilds)] async fn mute(ctx: &Context, msg: &Message) -> CommandResult { - let guild = msg.guild(&ctx.cache).unwrap(); - let guild_id = guild.id; + let guild_id = msg.guild(&ctx.cache).unwrap().id; let manager = songbird::get(ctx) .await @@ -343,8 +364,9 @@ async fn play_fade(ctx: &Context, msg: &Message, mut args: Args) -> CommandResul return Ok(()); } - let guild = msg.guild(&ctx.cache).unwrap(); - let guild_id = guild.id; + let guild_id = msg.guild_id.unwrap(); + + let http_client = get_http_client(ctx).await; let manager = songbird::get(ctx) .await @@ -354,20 +376,11 @@ async fn play_fade(ctx: &Context, msg: &Message, mut args: Args) -> CommandResul if let Some(handler_lock) = manager.get(guild_id) { let mut handler = handler_lock.lock().await; - let source = match input::ytdl(&url).await { - Ok(source) => source, - Err(why) => { - println!("Err starting source: {:?}", why); - - check_msg(msg.channel_id.say(&ctx.http, "Error sourcing ffmpeg").await); - - return Ok(()); - }, - }; + let src = YoutubeDl::new(http_client, url); // This handler object will allow you to, as needed, // control the audio track via events and further commands. - let song = handler.play_source(source); + let song = handler.play_input(src.into()); let send_http = ctx.http.clone(); let chan_id = msg.channel_id; @@ -474,8 +487,9 @@ async fn queue(ctx: &Context, msg: &Message, mut args: Args) -> CommandResult { return Ok(()); } - let guild = msg.guild(&ctx.cache).unwrap(); - let guild_id = guild.id; + let guild_id = msg.guild_id.unwrap(); + + let http_client = get_http_client(ctx).await; let manager = songbird::get(ctx) .await @@ -487,18 +501,9 @@ async fn queue(ctx: &Context, msg: &Message, mut args: Args) -> CommandResult { // Here, we use lazy restartable sources to make sure that we don't pay // for decoding, playback on tracks which aren't actually live yet. - let source = match Restartable::ytdl(url, true).await { - Ok(source) => source, - Err(why) => { - println!("Err starting source: {:?}", why); + let src = YoutubeDl::new(http_client, url); - check_msg(msg.channel_id.say(&ctx.http, "Error sourcing ffmpeg").await); - - return Ok(()); - }, - }; - - handler.enqueue_source(source.into()); + handler.enqueue_input(src.into()).await; check_msg( msg.channel_id @@ -522,8 +527,7 @@ async fn queue(ctx: &Context, msg: &Message, mut args: Args) -> CommandResult { #[command] #[only_in(guilds)] async fn skip(ctx: &Context, msg: &Message, _args: Args) -> CommandResult { - let guild = msg.guild(&ctx.cache).unwrap(); - let guild_id = guild.id; + let guild_id = msg.guild_id.unwrap(); let manager = songbird::get(ctx) .await @@ -557,8 +561,7 @@ async fn skip(ctx: &Context, msg: &Message, _args: Args) -> CommandResult { #[command] #[only_in(guilds)] async fn stop(ctx: &Context, msg: &Message, _args: Args) -> CommandResult { - let guild = msg.guild(&ctx.cache).unwrap(); - let guild_id = guild.id; + let guild_id = msg.guild_id.unwrap(); let manager = songbird::get(ctx) .await @@ -568,7 +571,7 @@ async fn stop(ctx: &Context, msg: &Message, _args: Args) -> CommandResult { if let Some(handler_lock) = manager.get(guild_id) { let handler = handler_lock.lock().await; let queue = handler.queue(); - let _ = queue.stop(); + queue.stop(); check_msg(msg.channel_id.say(&ctx.http, "Queue cleared.").await); } else { @@ -585,8 +588,7 @@ async fn stop(ctx: &Context, msg: &Message, _args: Args) -> CommandResult { #[command] #[only_in(guilds)] async fn undeafen(ctx: &Context, msg: &Message) -> CommandResult { - let guild = msg.guild(&ctx.cache).unwrap(); - let guild_id = guild.id; + let guild_id = msg.guild_id.unwrap(); let manager = songbird::get(ctx) .await @@ -618,8 +620,7 @@ async fn undeafen(ctx: &Context, msg: &Message) -> CommandResult { #[command] #[only_in(guilds)] async fn unmute(ctx: &Context, msg: &Message) -> CommandResult { - let guild = msg.guild(&ctx.cache).unwrap(); - let guild_id = guild.id; + let guild_id = msg.guild_id.unwrap(); let manager = songbird::get(ctx) .await .expect("Songbird Voice client placed in at initialisation.") diff --git a/examples/serenity/voice_receive/Cargo.toml b/examples/serenity/voice_receive/Cargo.toml index a7af106..c616efb 100644 --- a/examples/serenity/voice_receive/Cargo.toml +++ b/examples/serenity/voice_receive/Cargo.toml @@ -2,7 +2,7 @@ name = "voice_receive" version = "0.1.0" authors = ["my name "] -edition = "2018" +edition = "2021" [dependencies] tracing = "0.1" diff --git a/examples/serenity/voice_receive/src/main.rs b/examples/serenity/voice_receive/src/main.rs index bff1dbb..0d4d907 100644 --- a/examples/serenity/voice_receive/src/main.rs +++ b/examples/serenity/voice_receive/src/main.rs @@ -12,17 +12,14 @@ use serenity::{ async_trait, client::{Client, Context, EventHandler}, framework::{ - StandardFramework, standard::{ macros::{command, group}, - Args, CommandResult, + Args, + CommandResult, }, + StandardFramework, }, - model::{ - channel::Message, - gateway::Ready, - id::ChannelId, - }, + model::{channel::Message, gateway::Ready, id::ChannelId}, prelude::{GatewayIntents, Mentionable}, Result as SerenityResult, }; @@ -53,7 +50,7 @@ impl Receiver { pub fn new() -> Self { // You can manage state here, such as a buffer of audio packet bytes so // you can later store them in intervals. - Self { } + Self {} } } @@ -63,9 +60,12 @@ impl VoiceEventHandler for Receiver { async fn act(&self, ctx: &EventContext<'_>) -> Option { use EventContext as Ctx; match ctx { - Ctx::SpeakingStateUpdate( - Speaking {speaking, ssrc, user_id, ..} - ) => { + Ctx::SpeakingStateUpdate(Speaking { + speaking, + ssrc, + user_id, + .. + }) => { // Discord voice calls use RTP, where every sender uses a randomly allocated // *Synchronisation Source* (SSRC) to allow receivers to tell which audio // stream a received packet belongs to. As this number is not derived from @@ -79,9 +79,7 @@ impl VoiceEventHandler for Receiver { // to the user ID and handle their audio packets separately. println!( "Speaking state update: user {:?} has SSRC {:?}, using {:?}", - user_id, - ssrc, - speaking, + user_id, ssrc, speaking, ); }, Ctx::SpeakingUpdate(data) => { @@ -90,14 +88,17 @@ impl VoiceEventHandler for Receiver { println!( "Source {} has {} speaking.", data.ssrc, - if data.speaking {"started"} else {"stopped"}, + if data.speaking { "started" } else { "stopped" }, ); }, Ctx::VoicePacket(data) => { // An event which fires for every received audio packet, // containing the decoded data. if let Some(audio) = data.audio { - println!("Audio packet's first 5 samples: {:?}", audio.get(..5.min(audio.len()))); + println!( + "Audio packet's first 5 samples: {:?}", + audio.get(..5.min(audio.len())) + ); println!( "Audio packet sequence {:05} has {:04} bytes (decompressed from {}), SSRC {}", data.packet.sequence.0, @@ -114,9 +115,7 @@ impl VoiceEventHandler for Receiver { // containing the call statistics and reporting information. println!("RTCP packet received: {:?}", data.packet); }, - Ctx::ClientDisconnect( - ClientDisconnect {user_id, ..} - ) => { + Ctx::ClientDisconnect(ClientDisconnect { user_id, .. }) => { // You can implement your own logic here to handle a user who has left the // voice channel e.g., finalise processing of statistics etc. // You will typically need to map the User ID to their SSRC; observed when @@ -127,7 +126,7 @@ impl VoiceEventHandler for Receiver { _ => { // We won't be registering this struct for any more event classes. unimplemented!() - } + }, } None @@ -141,24 +140,20 @@ struct General; #[tokio::main] async fn main() { tracing_subscriber::fmt::init(); - + // Configure the client with your Discord bot token in the environment. - let token = env::var("DISCORD_TOKEN") - .expect("Expected a token in the environment"); + let token = env::var("DISCORD_TOKEN").expect("Expected a token in the environment"); let framework = StandardFramework::new() - .configure(|c| c - .prefix("~")) .group(&GENERAL_GROUP); + framework.configure(|c| c.prefix("~")); - let intents = GatewayIntents::non_privileged() - | GatewayIntents::MESSAGE_CONTENT; + let intents = GatewayIntents::non_privileged() | GatewayIntents::MESSAGE_CONTENT; // Here, we need to configure Songbird to decode all incoming voice packets. // If you want, you can do this on a per-call basis---here, we need it to // read the audio data that other people are sending us! - let songbird_config = Config::default() - .decode_mode(DecodeMode::Decode); + let songbird_config = Config::default().decode_mode(DecodeMode::Decode); let mut client = Client::builder(&token, intents) .event_handler(Handler) @@ -167,26 +162,33 @@ async fn main() { .await .expect("Err creating client"); - let _ = client.start().await.map_err(|why| println!("Client ended: {:?}", why)); + let _ = client + .start() + .await + .map_err(|why| println!("Client ended: {:?}", why)); } #[command] #[only_in(guilds)] async fn join(ctx: &Context, msg: &Message, mut args: Args) -> CommandResult { - let connect_to = match args.single::() { + let connect_to = match args.single::() { Ok(id) => ChannelId(id), Err(_) => { - check_msg(msg.reply(ctx, "Requires a valid voice channel ID be given").await); + check_msg( + msg.reply(ctx, "Requires a valid voice channel ID be given") + .await, + ); return Ok(()); }, }; - let guild = msg.guild(&ctx.cache).unwrap(); - let guild_id = guild.id; + let guild_id = msg.guild_id.unwrap(); - let manager = songbird::get(ctx).await - .expect("Songbird Voice client placed in at initialisation.").clone(); + let manager = songbird::get(ctx) + .await + .expect("Songbird Voice client placed in at initialisation.") + .clone(); let (handler_lock, conn_result) = manager.join(guild_id, connect_to).await; @@ -194,34 +196,27 @@ async fn join(ctx: &Context, msg: &Message, mut args: Args) -> CommandResult { // NOTE: this skips listening for the actual connection result. let mut handler = handler_lock.lock().await; - handler.add_global_event( - CoreEvent::SpeakingStateUpdate.into(), - Receiver::new(), - ); + handler.add_global_event(CoreEvent::SpeakingStateUpdate.into(), Receiver::new()); - handler.add_global_event( - CoreEvent::SpeakingUpdate.into(), - Receiver::new(), - ); + handler.add_global_event(CoreEvent::SpeakingUpdate.into(), Receiver::new()); - handler.add_global_event( - CoreEvent::VoicePacket.into(), - Receiver::new(), - ); + handler.add_global_event(CoreEvent::VoicePacket.into(), Receiver::new()); - handler.add_global_event( - CoreEvent::RtcpPacket.into(), - Receiver::new(), - ); + handler.add_global_event(CoreEvent::RtcpPacket.into(), Receiver::new()); - handler.add_global_event( - CoreEvent::ClientDisconnect.into(), - Receiver::new(), - ); + handler.add_global_event(CoreEvent::ClientDisconnect.into(), Receiver::new()); - check_msg(msg.channel_id.say(&ctx.http, &format!("Joined {}", connect_to.mention())).await); + check_msg( + msg.channel_id + .say(&ctx.http, &format!("Joined {}", connect_to.mention())) + .await, + ); } else { - check_msg(msg.channel_id.say(&ctx.http, "Error joining the channel").await); + check_msg( + msg.channel_id + .say(&ctx.http, "Error joining the channel") + .await, + ); } Ok(()) @@ -230,19 +225,24 @@ async fn join(ctx: &Context, msg: &Message, mut args: Args) -> CommandResult { #[command] #[only_in(guilds)] async fn leave(ctx: &Context, msg: &Message) -> CommandResult { - let guild = msg.guild(&ctx.cache).unwrap(); - let guild_id = guild.id; + let guild_id = msg.guild_id.unwrap(); - let manager = songbird::get(ctx).await - .expect("Songbird Voice client placed in at initialisation.").clone(); + let manager = songbird::get(ctx) + .await + .expect("Songbird Voice client placed in at initialisation.") + .clone(); let has_handler = manager.get(guild_id).is_some(); if has_handler { if let Err(e) = manager.remove(guild_id).await { - check_msg(msg.channel_id.say(&ctx.http, format!("Failed: {:?}", e)).await); + check_msg( + msg.channel_id + .say(&ctx.http, format!("Failed: {:?}", e)) + .await, + ); } - check_msg(msg.channel_id.say(&ctx.http,"Left voice channel").await); + check_msg(msg.channel_id.say(&ctx.http, "Left voice channel").await); } else { check_msg(msg.reply(ctx, "Not in a voice channel").await); } @@ -252,7 +252,7 @@ async fn leave(ctx: &Context, msg: &Message) -> CommandResult { #[command] async fn ping(ctx: &Context, msg: &Message) -> CommandResult { - check_msg(msg.channel_id.say(&ctx.http,"Pong!").await); + check_msg(msg.channel_id.say(&ctx.http, "Pong!").await); Ok(()) } diff --git a/examples/twilight/Cargo.toml b/examples/twilight/Cargo.toml index de8e482..d00cb1f 100644 --- a/examples/twilight/Cargo.toml +++ b/examples/twilight/Cargo.toml @@ -2,10 +2,11 @@ name = "twilight" version = "0.1.0" authors = ["Twilight and Serenity Contributors"] -edition = "2018" +edition = "2021" [dependencies] futures = "0.3" +reqwest = "0.11" tracing = "0.1" tracing-subscriber = "0.2" tokio = { features = ["macros", "rt-multi-thread", "sync"], version = "1" } @@ -18,3 +19,9 @@ twilight-standby = "0.12" default-features = false path = "../.." features = ["driver", "twilight-rustls", "zlib-stock"] + +[dependencies.symphonia] +version = "0.5" +features = ["aac", "mp3", "isomp4", "alac"] +git = "https://github.com/FelixMcFelix/Symphonia" +branch = "songbird-fixes" diff --git a/examples/twilight/src/main.rs b/examples/twilight/src/main.rs index aab0569..d50ea2e 100644 --- a/examples/twilight/src/main.rs +++ b/examples/twilight/src/main.rs @@ -22,11 +22,11 @@ use futures::StreamExt; use songbird::{ - input::{Input, Restartable}, + input::{Compose, YoutubeDl}, tracks::{PlayMode, TrackHandle}, Songbird, }; -use std::{collections::HashMap, env, error::Error, future::Future, sync::Arc}; +use std::{collections::HashMap, env, error::Error, future::Future, num::NonZeroU64, sync::Arc}; use tokio::sync::RwLock; use twilight_gateway::{Cluster, Event, Intents}; use twilight_http::Client as HttpClient; @@ -68,7 +68,7 @@ async fn main() -> Result<(), Box> { let http = HttpClient::new(token.clone()); let user_id = http.current_user().exec().await?.model().await?.id; - let intents = Intents::GUILD_MESSAGES | Intents::GUILD_VOICE_STATES; + let intents = Intents::GUILD_MESSAGES | Intents::MESSAGE_CONTENT | Intents::GUILD_VOICE_STATES; let (cluster, events) = Cluster::new(token, intents).await?; cluster.up().await; @@ -81,8 +81,8 @@ async fn main() -> Result<(), Box> { trackdata: Default::default(), songbird, standby: Standby::new(), - }, - )) + }), + ) }; while let Some((_, event)) = events.next().await { @@ -128,6 +128,8 @@ async fn join(msg: Message, state: State) -> Result<(), Box()?; let guild_id = msg.guild_id.ok_or("Can't join a non-guild channel.")?; + let channel_id = + NonZeroU64::new(channel_id).ok_or("Joined voice channel must have nonzero ID.")?; let (_handle, success) = state.songbird.join(guild_id, channel_id).await; @@ -190,21 +192,12 @@ async fn play(msg: Message, state: State) -> Result<(), Box".to_string()), - input - .metadata - .artist - .as_ref() - .unwrap_or(&"".to_string()), + metadata.track.as_ref().unwrap_or(&"".to_string()), + metadata.artist.as_ref().unwrap_or(&"".to_string()), ); state @@ -216,7 +209,7 @@ async fn play(msg: Message, state: State) -> Result<(), Box Result<(), Box { let _success = handle.pause(); false - } + }, _ => { let _success = handle.play(); true - } + }, }; let action = if paused { "Unpaused" } else { "Paused" }; @@ -301,12 +294,8 @@ async fn seek(msg: Message, state: State) -> Result<(), Box - + @@ -49,6 +49,40 @@ Core + + + + + + Thread Pool + + + + + + + + Seek + + + + + + + + Compose + + + + + + + + ... + + + + @@ -113,6 +147,9 @@ + + + diff --git a/images/gateway.png b/images/gateway.png index 0916044..d3e91d2 100644 Binary files a/images/gateway.png and b/images/gateway.png differ diff --git a/resources/Cloudkicker - 2011 07.dca1 b/resources/Cloudkicker - 2011 07.dca1 new file mode 100644 index 0000000..2f3ee7d Binary files /dev/null and b/resources/Cloudkicker - 2011 07.dca1 differ diff --git a/examples/serenity/voice_storage/Cloudkicker_-_Loops_-_22_2011_07.mp3 b/resources/Cloudkicker - 2011 07.mp3 similarity index 100% rename from examples/serenity/voice_storage/Cloudkicker_-_Loops_-_22_2011_07.mp3 rename to resources/Cloudkicker - 2011 07.mp3 diff --git a/resources/Cloudkicker - Making Will Mad.opus b/resources/Cloudkicker - Making Will Mad.opus new file mode 100644 index 0000000..2fb78ba Binary files /dev/null and b/resources/Cloudkicker - Making Will Mad.opus differ diff --git a/resources/Cloudkicker - Making Will Mad.webm b/resources/Cloudkicker - Making Will Mad.webm new file mode 100644 index 0000000..631a316 Binary files /dev/null and b/resources/Cloudkicker - Making Will Mad.webm differ diff --git a/resources/README.md b/resources/README.md new file mode 100644 index 0000000..36f928c --- /dev/null +++ b/resources/README.md @@ -0,0 +1,8 @@ +# Resources +This folder contains various audio files used for testing or within examples. + +Some songs are used under creative commons licenses ([Ben Sharp/Cloudkicker](https://cloudkicker.bandcamp.com/)): +* `Cloudkicker - 2011 07.[mp3,dca1]` – [CC BY-NC-SA 3.0](https://creativecommons.org/licenses/by-nc-sa/3.0/) +* `Cloudkicker - Making Will Mad.[opus,webm]` – [CC BY 3.0](https://creativecommons.org/licenses/by/3.0/) + +All sound files are made by contributors. diff --git a/resources/loop-48.mp3 b/resources/loop-48.mp3 new file mode 100644 index 0000000..814591f Binary files /dev/null and b/resources/loop-48.mp3 differ diff --git a/examples/serenity/voice_storage/loop.wav b/resources/loop.wav similarity index 100% rename from examples/serenity/voice_storage/loop.wav rename to resources/loop.wav diff --git a/resources/ting.mp3 b/resources/ting.mp3 new file mode 100644 index 0000000..3eac893 Binary files /dev/null and b/resources/ting.mp3 differ diff --git a/examples/serenity/voice_storage/ting.wav b/resources/ting.wav similarity index 100% rename from examples/serenity/voice_storage/ting.wav rename to resources/ting.wav diff --git a/src/config.rs b/src/config.rs index 692a855..2265ca8 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,13 +1,24 @@ -#[cfg(feature = "driver-core")] -use super::driver::{retry::Retry, CryptoMode, DecodeMode}; +#[cfg(feature = "driver")] +use crate::{ + driver::{retry::Retry, CryptoMode, DecodeMode, MixMode}, + input::codecs::*, +}; +#[cfg(test)] +use crate::driver::test_config::*; + +#[cfg(feature = "driver")] +use symphonia::core::{codecs::CodecRegistry, probe::Probe}; + +use derivative::Derivative; use std::time::Duration; /// Configuration for drivers and calls. -#[derive(Clone, Debug)] +#[derive(Clone, Derivative)] +#[derivative(Debug)] #[non_exhaustive] pub struct Config { - #[cfg(feature = "driver-core")] + #[cfg(feature = "driver")] /// Selected tagging mode for voice packet encryption. /// /// Defaults to [`CryptoMode::Normal`]. @@ -18,7 +29,7 @@ pub struct Config { /// /// [`CryptoMode::Normal`]: CryptoMode::Normal pub crypto_mode: CryptoMode, - #[cfg(feature = "driver-core")] + #[cfg(feature = "driver")] /// Configures whether decoding and decryption occur for all received packets. /// /// If voice receiving voice packets, generally you should choose [`DecodeMode::Decode`]. @@ -34,7 +45,7 @@ pub struct Config { /// [`DecodeMode::Pass`]: DecodeMode::Pass /// [user speaking events]: crate::events::CoreEvent::SpeakingUpdate pub decode_mode: DecodeMode, - #[cfg(feature = "gateway-core")] + #[cfg(feature = "gateway")] /// Configures the amount of time to wait for Discord to reply with connection information /// if [`Call::join`]/[`join_gateway`] are used. /// @@ -47,7 +58,15 @@ pub struct Config { /// [`Call::join`]: crate::Call::join /// [`join_gateway`]: crate::Call::join_gateway pub gateway_timeout: Option, - #[cfg(feature = "driver-core")] + #[cfg(feature = "driver")] + /// Configures the maximum amount of time to wait for an attempted voice + /// connection to Discord. + /// + /// Defaults to [`Stereo`]. + /// + /// [`Stereo`]: MixMode::Stereo + pub mix_mode: MixMode, + #[cfg(feature = "driver")] /// Number of concurrently active tracks to allocate memory for. /// /// This should be set at, or just above, the maximum number of tracks @@ -60,7 +79,7 @@ pub struct Config { /// Changes to this field in a running driver will only ever increase /// the capacity of the track store. pub preallocated_tracks: usize, - #[cfg(feature = "driver-core")] + #[cfg(feature = "driver")] /// Connection retry logic for the [`Driver`]. /// /// This controls how many times the [`Driver`] should retry any connections, @@ -68,65 +87,131 @@ pub struct Config { /// /// [`Driver`]: crate::driver::Driver pub driver_retry: Retry, - #[cfg(feature = "driver-core")] + #[cfg(feature = "driver")] /// Configures the maximum amount of time to wait for an attempted voice /// connection to Discord. /// /// Defaults to 10 seconds. If set to `None`, connections will never time out. pub driver_timeout: Option, + #[cfg(feature = "driver")] + #[derivative(Debug = "ignore")] + /// Registry of the inner codecs supported by the driver, adding audiopus-based + /// Opus codec support to all of Symphonia's default codecs. + /// + /// Defaults to [`CODEC_REGISTRY`]. + /// + /// [`CODEC_REGISTRY`]: static@CODEC_REGISTRY + pub codec_registry: &'static CodecRegistry, + #[cfg(feature = "driver")] + #[derivative(Debug = "ignore")] + /// Registry of the muxers and container formats supported by the driver. + /// + /// Defaults to [`PROBE`], which includes all of Symphonia's default format handlers + /// and DCA format support. + /// + /// [`PROBE`]: static@PROBE + pub format_registry: &'static Probe, + + // Test only attributes + #[cfg(feature = "driver")] + #[cfg(test)] + /// Test config to offer precise control over mixing tick rate/count. + pub(crate) tick_style: TickStyle, + #[cfg(feature = "driver")] + #[cfg(test)] + /// If set, skip connection and encryption steps. + pub(crate) override_connection: Option, } impl Default for Config { fn default() -> Self { Self { - #[cfg(feature = "driver-core")] + #[cfg(feature = "driver")] crypto_mode: CryptoMode::Normal, - #[cfg(feature = "driver-core")] + #[cfg(feature = "driver")] decode_mode: DecodeMode::Decrypt, - #[cfg(feature = "gateway-core")] + #[cfg(feature = "gateway")] gateway_timeout: Some(Duration::from_secs(10)), - #[cfg(feature = "driver-core")] + #[cfg(feature = "driver")] + mix_mode: MixMode::Stereo, + #[cfg(feature = "driver")] preallocated_tracks: 1, - #[cfg(feature = "driver-core")] - driver_retry: Default::default(), - #[cfg(feature = "driver-core")] + #[cfg(feature = "driver")] + driver_retry: Retry::default(), + #[cfg(feature = "driver")] driver_timeout: Some(Duration::from_secs(10)), + #[cfg(feature = "driver")] + codec_registry: &CODEC_REGISTRY, + #[cfg(feature = "driver")] + format_registry: &PROBE, + #[cfg(feature = "driver")] + #[cfg(test)] + tick_style: TickStyle::Timed, + #[cfg(feature = "driver")] + #[cfg(test)] + override_connection: None, } } } -#[cfg(feature = "driver-core")] +#[cfg(feature = "driver")] impl Config { /// Sets this `Config`'s chosen cryptographic tagging scheme. + #[must_use] pub fn crypto_mode(mut self, crypto_mode: CryptoMode) -> Self { self.crypto_mode = crypto_mode; self } /// Sets this `Config`'s received packet decryption/decoding behaviour. + #[must_use] pub fn decode_mode(mut self, decode_mode: DecodeMode) -> Self { self.decode_mode = decode_mode; self } + /// Sets this `Config`'s audio mixing channel count. + #[must_use] + pub fn mix_mode(mut self, mix_mode: MixMode) -> Self { + self.mix_mode = mix_mode; + self + } + /// Sets this `Config`'s number of tracks to preallocate. + #[must_use] pub fn preallocated_tracks(mut self, preallocated_tracks: usize) -> Self { self.preallocated_tracks = preallocated_tracks; self } /// Sets this `Config`'s timeout for establishing a voice connection. + #[must_use] pub fn driver_timeout(mut self, driver_timeout: Option) -> Self { self.driver_timeout = driver_timeout; self } /// Sets this `Config`'s voice connection retry configuration. + #[must_use] pub fn driver_retry(mut self, driver_retry: Retry) -> Self { self.driver_retry = driver_retry; self } + /// Sets this `Config`'s symphonia codec registry. + #[must_use] + pub fn codec_registry(mut self, codec_registry: &'static CODEC_REGISTRY) -> Self { + self.codec_registry = codec_registry; + self + } + + /// Sets this `Config`'s symphonia format registry/probe set. + #[must_use] + pub fn format_registry(mut self, format_registry: &'static PROBE) -> Self { + self.format_registry = format_registry; + self + } + /// This is used to prevent changes which would invalidate the current session. pub(crate) fn make_safe(&mut self, previous: &Config, connected: bool) { if connected { @@ -135,9 +220,50 @@ impl Config { } } -#[cfg(feature = "gateway-core")] +// Test only attributes +#[cfg(all(test, feature = "driver"))] +impl Config { + #![allow(missing_docs)] + #[must_use] + pub fn tick_style(mut self, tick_style: TickStyle) -> Self { + self.tick_style = tick_style; + self + } + + /// Sets this `Config`'s voice connection retry configuration. + #[must_use] + pub fn override_connection(mut self, override_connection: Option) -> Self { + self.override_connection = override_connection; + self + } + + pub fn test_cfg(raw_output: bool) -> (DriverTestHandle, Config) { + let (tick_tx, tick_rx) = flume::unbounded(); + + let (conn, rx) = if raw_output { + let (pkt_tx, pkt_rx) = flume::unbounded(); + + (OutputMode::Raw(pkt_tx), OutputReceiver::Raw(pkt_rx)) + } else { + let (rtp_tx, rtp_rx) = flume::unbounded(); + + (OutputMode::Rtp(rtp_tx), OutputReceiver::Rtp(rtp_rx)) + }; + + let config = Config::default() + .tick_style(TickStyle::UntimedWithExecLimit(tick_rx)) + .override_connection(Some(conn)); + + let handle = DriverTestHandle { rx, tx: tick_tx }; + + (handle, config) + } +} + +#[cfg(feature = "gateway")] impl Config { /// Sets this `Config`'s timeout for joining a voice channel. + #[must_use] pub fn gateway_timeout(mut self, gateway_timeout: Option) -> Self { self.gateway_timeout = gateway_timeout; self diff --git a/src/constants.rs b/src/constants.rs index 9c2e834..bf7e6d5 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -1,16 +1,16 @@ //! Constants affecting driver function and API handling. -#[cfg(feature = "driver-core")] +#[cfg(feature = "driver")] use audiopus::{Bitrate, SampleRate}; -#[cfg(feature = "driver-core")] +#[cfg(feature = "driver")] use discortp::rtp::RtpType; use std::time::Duration; -#[cfg(feature = "driver-core")] +#[cfg(feature = "driver")] /// The voice gateway version used by the library. pub const VOICE_GATEWAY_VERSION: u8 = crate::model::constants::GATEWAY_VERSION; -#[cfg(feature = "driver-core")] +#[cfg(feature = "driver")] /// Sample rate of audio to be sent to Discord. pub const SAMPLE_RATE: SampleRate = SampleRate::Hz48000; @@ -23,10 +23,16 @@ pub const AUDIO_FRAME_RATE: usize = 50; /// Length of time between any two audio frames. pub const TIMESTEP_LENGTH: Duration = Duration::from_millis(1000 / AUDIO_FRAME_RATE as u64); -#[cfg(feature = "driver-core")] +#[cfg(feature = "driver")] /// Default bitrate for audio. pub const DEFAULT_BITRATE: Bitrate = Bitrate::BitsPerSecond(128_000); +/// Number of output samples at 48kHZ to produced when resampling subframes. +pub(crate) const RESAMPLE_OUTPUT_FRAME_SIZE: usize = MONO_FRAME_SIZE / 2; + +/// The maximum number of bad frames to allow in an Opus source before blocking passthrough. +pub(crate) const OPUS_PASSTHROUGH_STRIKE_LIMIT: u8 = 3; + /// Number of samples in one complete frame of audio per channel. /// /// This is equally the number of stereo (joint) samples in an audio frame. @@ -70,6 +76,42 @@ pub const SILENT_FRAME: [u8; 3] = [0xf8, 0xff, 0xfe]; /// The one (and only) RTP version. pub const RTP_VERSION: u8 = 2; -#[cfg(feature = "driver-core")] +#[cfg(feature = "driver")] /// Profile type used by Discord's Opus audio traffic. pub const RTP_PROFILE_TYPE: RtpType = RtpType::Dynamic(120); + +#[cfg(test)] +pub mod test_data { + /// URL for a source which YTDL must extract. + /// + /// Referenced under CC BY-NC-SA 3.0 -- https://creativecommons.org/licenses/by-nc-sa/3.0/ + pub const YTDL_TARGET: &str = "https://cloudkicker.bandcamp.com/track/94-days"; + + /// URL for a source which can be read via an Http Request. + /// + /// Referenced under CC BY-NC-SA 3.0 -- https://creativecommons.org/licenses/by-nc-sa/3.0/ + pub const HTTP_TARGET: &str = "https://github.com/FelixMcFelix/songbird/raw/symphonia/resources/Cloudkicker%20-%202011%2007.mp3"; + + /// URL for an opus/ogg source which can be read via an Http Request. + /// + /// Referenced under CC BY 3.0 -- https://creativecommons.org/licenses/by/3.0/ + pub const HTTP_OPUS_TARGET: &str = "https://github.com/FelixMcFelix/songbird/raw/symphonia/resources/Cloudkicker%20-%20Making%20Will%20Mad.opus"; + + /// URL for an opus/webm source which can be read via an Http Request. + /// + /// Referenced under CC BY 3.0 -- https://creativecommons.org/licenses/by/3.0/ + pub const HTTP_WEBM_TARGET: &str = "https://github.com/FelixMcFelix/songbird/raw/symphonia/resources/Cloudkicker%20-%20Making%20Will%20Mad.webm"; + + /// Path to a DCA source. + /// + /// Referenced under CC BY-NC-SA 3.0 -- https://creativecommons.org/licenses/by-nc-sa/3.0/ + pub const FILE_DCA_TARGET: &str = "resources/Cloudkicker - 2011 07.dca1"; + + /// Path to an opus source which can be read via a File. + /// + /// Referenced under CC BY 3.0 -- https://creativecommons.org/licenses/by/3.0/ + pub const FILE_WEBM_TARGET: &str = "resources/Cloudkicker - Making Will Mad.webm"; + + /// Path to a Wav source which can be read via a File. + pub const FILE_WAV_TARGET: &str = "resources/loop.wav"; +} diff --git a/src/driver/bench_internals.rs b/src/driver/bench_internals.rs index d335d49..a12fa97 100644 --- a/src/driver/bench_internals.rs +++ b/src/driver/bench_internals.rs @@ -1,8 +1,19 @@ -//! Various driver internals which need to be exported for benchmarking. +//! Various driver internals which need to be exported for benchmarking. //! //! Included if using the `"internals"` feature flag. //! You should not and/or cannot use these as part of a normal application. +#![allow(missing_docs)] + pub use super::tasks::{message as task_message, mixer}; pub use super::crypto::CryptoState; + +use crate::{ + driver::tasks::message::TrackContext, + tracks::{Track, TrackHandle}, +}; + +pub fn track_context(t: Track) -> (TrackHandle, TrackContext) { + t.into_context() +} diff --git a/src/driver/connection/error.rs b/src/driver/connection/error.rs index 0dc23d3..3b01eb6 100644 --- a/src/driver/connection/error.rs +++ b/src/driver/connection/error.rs @@ -94,21 +94,21 @@ impl From for Error { impl fmt::Display for Error { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "failed to connect to Discord RTP server: ")?; - use Error::*; match self { - AttemptDiscarded => write!(f, "connection attempt was aborted/discarded"), - Crypto(e) => e.fmt(f), - CryptoModeInvalid => write!(f, "server changed negotiated encryption mode"), - CryptoModeUnavailable => write!(f, "server did not offer chosen encryption mode"), - EndpointUrl => write!(f, "endpoint URL received from gateway was invalid"), - ExpectedHandshake => write!(f, "voice initialisation protocol was violated"), - IllegalDiscoveryResponse => write!(f, "IP discovery/NAT punching response was invalid"), - IllegalIp => write!(f, "IP discovery/NAT punching response had bad IP value"), - Io(e) => e.fmt(f), - Json(e) => e.fmt(f), - InterconnectFailure(e) => write!(f, "failed to contact other task ({:?})", e), - Ws(e) => write!(f, "websocket issue ({:?}).", e), - TimedOut => write!(f, "connection attempt timed out"), + Self::AttemptDiscarded => write!(f, "connection attempt was aborted/discarded"), + Self::Crypto(e) => e.fmt(f), + Self::CryptoModeInvalid => write!(f, "server changed negotiated encryption mode"), + Self::CryptoModeUnavailable => write!(f, "server did not offer chosen encryption mode"), + Self::EndpointUrl => write!(f, "endpoint URL received from gateway was invalid"), + Self::ExpectedHandshake => write!(f, "voice initialisation protocol was violated"), + Self::IllegalDiscoveryResponse => + write!(f, "IP discovery/NAT punching response was invalid"), + Self::IllegalIp => write!(f, "IP discovery/NAT punching response had bad IP value"), + Self::Io(e) => e.fmt(f), + Self::Json(e) => e.fmt(f), + Self::InterconnectFailure(e) => write!(f, "failed to contact other task ({:?})", e), + Self::Ws(e) => write!(f, "websocket issue ({:?}).", e), + Self::TimedOut => write!(f, "connection attempt timed out"), } } } @@ -116,19 +116,19 @@ impl fmt::Display for Error { impl StdError for Error { fn source(&self) -> Option<&(dyn StdError + 'static)> { match self { - Error::AttemptDiscarded => None, + Error::AttemptDiscarded + | Error::CryptoModeInvalid + | Error::CryptoModeUnavailable + | Error::EndpointUrl + | Error::ExpectedHandshake + | Error::IllegalDiscoveryResponse + | Error::IllegalIp + | Error::InterconnectFailure(_) + | Error::Ws(_) + | Error::TimedOut => None, Error::Crypto(e) => e.source(), - Error::CryptoModeInvalid => None, - Error::CryptoModeUnavailable => None, - Error::EndpointUrl => None, - Error::ExpectedHandshake => None, - Error::IllegalDiscoveryResponse => None, - Error::IllegalIp => None, Error::Io(e) => e.source(), Error::Json(e) => e.source(), - Error::InterconnectFailure(_) => None, - Error::Ws(_) => None, - Error::TimedOut => None, } } } diff --git a/src/driver/crypto.rs b/src/driver/crypto.rs index 18e408b..d127950 100644 --- a/src/driver/crypto.rs +++ b/src/driver/crypto.rs @@ -12,7 +12,7 @@ use xsalsa20poly1305::{ TAG_SIZE, }; -/// Variants of the XSalsa20Poly1305 encryption scheme. +/// Variants of the `XSalsa20Poly1305` encryption scheme. #[derive(Clone, Copy, Debug, Eq, PartialEq)] #[non_exhaustive] pub enum CryptoMode { @@ -35,57 +35,58 @@ pub enum CryptoMode { impl From for CryptoMode { fn from(val: CryptoState) -> Self { - use CryptoState::*; match val { - Normal => CryptoMode::Normal, - Suffix => CryptoMode::Suffix, - Lite(_) => CryptoMode::Lite, + CryptoState::Normal => Self::Normal, + CryptoState::Suffix => Self::Suffix, + CryptoState::Lite(_) => Self::Lite, } } } impl CryptoMode { /// Returns the name of a mode as it will appear during negotiation. + #[must_use] pub fn to_request_str(self) -> &'static str { - use CryptoMode::*; match self { - Normal => "xsalsa20_poly1305", - Suffix => "xsalsa20_poly1305_suffix", - Lite => "xsalsa20_poly1305_lite", + Self::Normal => "xsalsa20_poly1305", + Self::Suffix => "xsalsa20_poly1305_suffix", + Self::Lite => "xsalsa20_poly1305_lite", } } /// Returns the number of bytes each nonce is stored as within /// a packet. + #[must_use] pub fn nonce_size(self) -> usize { - use CryptoMode::*; match self { - Normal => RtpPacket::minimum_packet_size(), - Suffix => NONCE_SIZE, - Lite => 4, + Self::Normal => RtpPacket::minimum_packet_size(), + Self::Suffix => NONCE_SIZE, + Self::Lite => 4, } } /// Returns the number of bytes occupied by the encryption scheme /// which fall before the payload. - pub fn payload_prefix_len(self) -> usize { + #[must_use] + pub fn payload_prefix_len() -> usize { TAG_SIZE } /// Returns the number of bytes occupied by the encryption scheme /// which fall after the payload. + #[must_use] pub fn payload_suffix_len(self) -> usize { - use CryptoMode::*; match self { - Normal => 0, - Suffix | Lite => self.nonce_size(), + Self::Normal => 0, + Self::Suffix | Self::Lite => self.nonce_size(), } } /// Calculates the number of additional bytes required compared /// to an unencrypted payload. + #[must_use] pub fn payload_overhead(self) -> usize { - self.payload_prefix_len() + self.payload_suffix_len() + Self::payload_prefix_len() + self.payload_suffix_len() } /// Extracts the byte slice in a packet used as the nonce, and the remaining mutable @@ -95,10 +96,9 @@ impl CryptoMode { header: &'a [u8], body: &'a mut [u8], ) -> Result<(&'a [u8], &'a mut [u8]), CryptoError> { - use CryptoMode::*; match self { - Normal => Ok((header, body)), - Suffix | Lite => { + Self::Normal => Ok((header, body)), + Self::Suffix | Self::Lite => { let len = body.len(); if len < self.payload_suffix_len() { Err(CryptoError) @@ -135,7 +135,7 @@ impl CryptoMode { &nonce }; - let body_start = self.payload_prefix_len(); + let body_start = Self::payload_prefix_len(); let body_tail = self.payload_suffix_len(); if body_start > body_remaining.len() { @@ -183,22 +183,33 @@ impl CryptoMode { } } -#[allow(missing_docs)] +/// State used in nonce generation for the `XSalsa20Poly1305` encryption variants +/// in [`CryptoMode`]. #[derive(Clone, Copy, Debug, Eq, PartialEq)] #[non_exhaustive] pub enum CryptoState { + /// The RTP header is used as the source of nonce bytes for the packet. + /// + /// No state is required. Normal, + /// An additional random 24B suffix is used as the source of nonce bytes for the packet. + /// This is regenerated randomly for each packet. + /// + /// No state is required. Suffix, + /// An additional random 4B suffix is used as the source of nonce bytes for the packet. + /// This nonce value increments by `1` with each packet. + /// + /// The last used nonce is stored. Lite(Wrapping), } impl From for CryptoState { fn from(val: CryptoMode) -> Self { - use CryptoMode::*; match val { - Normal => CryptoState::Normal, - Suffix => CryptoState::Suffix, - Lite => CryptoState::Lite(Wrapping(rand::random::())), + CryptoMode::Normal => CryptoState::Normal, + CryptoMode::Suffix => CryptoState::Suffix, + CryptoMode::Lite => CryptoState::Lite(Wrapping(rand::random::())), } } } @@ -213,12 +224,11 @@ impl CryptoState { let mode = self.kind(); let endpoint = payload_end + mode.payload_suffix_len(); - use CryptoState::*; match self { - Suffix => { + Self::Suffix => { rand::thread_rng().fill(&mut packet.payload_mut()[payload_end..endpoint]); }, - Lite(mut i) => { + Self::Lite(mut i) => { (&mut packet.payload_mut()[payload_end..endpoint]) .write_u32::(i.0) .expect( @@ -233,8 +243,8 @@ impl CryptoState { } /// Returns the underlying (stateless) type of the active crypto mode. - pub fn kind(&self) -> CryptoMode { - CryptoMode::from(*self) + pub fn kind(self) -> CryptoMode { + CryptoMode::from(self) } } @@ -246,7 +256,7 @@ mod test { #[test] fn small_packet_decrypts_error() { - let mut buf = [0u8; MutableRtpPacket::minimum_packet_size() + 0]; + let mut buf = [0u8; MutableRtpPacket::minimum_packet_size()]; let modes = [CryptoMode::Normal, CryptoMode::Suffix, CryptoMode::Lite]; let mut pkt = MutableRtpPacket::new(&mut buf[..]).unwrap(); diff --git a/src/driver/decode_mode.rs b/src/driver/decode_mode.rs index 55c0389..1721592 100644 --- a/src/driver/decode_mode.rs +++ b/src/driver/decode_mode.rs @@ -26,6 +26,7 @@ pub enum DecodeMode { impl DecodeMode { /// Returns whether this mode will decrypt received packets. + #[must_use] pub fn should_decrypt(self) -> bool { self != DecodeMode::Pass } diff --git a/src/driver/mix_mode.rs b/src/driver/mix_mode.rs new file mode 100644 index 0000000..4fdcdd1 --- /dev/null +++ b/src/driver/mix_mode.rs @@ -0,0 +1,59 @@ +use audiopus::Channels; +use symphonia_core::audio::Layout; + +use crate::constants::{MONO_FRAME_SIZE, STEREO_FRAME_SIZE}; + +/// Mixing behaviour for sent audio sources processed within the driver. +/// +/// This has no impact on Opus packet passthrough, which will pass packets +/// irrespective of their channel count. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum MixMode { + /// Audio sources will be downmixed into a mono buffer. + Mono, + /// Audio sources will be mixed into into a stereo buffer, where mono sources + /// will be duplicated into both channels. + Stereo, +} + +impl MixMode { + pub(crate) const fn to_opus(self) -> Channels { + match self { + Self::Mono => Channels::Mono, + Self::Stereo => Channels::Stereo, + } + } + + pub(crate) const fn sample_count_in_frame(self) -> usize { + match self { + Self::Mono => MONO_FRAME_SIZE, + Self::Stereo => STEREO_FRAME_SIZE, + } + } + + pub(crate) const fn channels(self) -> usize { + match self { + Self::Mono => 1, + Self::Stereo => 2, + } + } + + pub(crate) const fn symph_layout(self) -> Layout { + match self { + Self::Mono => Layout::Mono, + Self::Stereo => Layout::Stereo, + } + } +} + +impl From for Layout { + fn from(val: MixMode) -> Self { + val.symph_layout() + } +} + +impl From for Channels { + fn from(val: MixMode) -> Self { + val.to_opus() + } +} diff --git a/src/driver/mod.rs b/src/driver/mod.rs index 4210b94..20a3f02 100644 --- a/src/driver/mod.rs +++ b/src/driver/mod.rs @@ -14,20 +14,26 @@ pub mod bench_internals; pub(crate) mod connection; mod crypto; mod decode_mode; +mod mix_mode; pub mod retry; pub(crate) mod tasks; +#[cfg(test)] +pub(crate) mod test_config; use connection::error::{Error, Result}; pub use crypto::CryptoMode; pub(crate) use crypto::CryptoState; pub use decode_mode::DecodeMode; +pub use mix_mode::MixMode; +#[cfg(test)] +pub use test_config::*; #[cfg(feature = "builtin-queue")] use crate::tracks::TrackQueue; use crate::{ events::EventData, input::Input, - tracks::{self, Track, TrackHandle}, + tracks::{Track, TrackHandle}, Config, ConnectionInfo, Event, @@ -41,6 +47,8 @@ use core::{ task::{Context, Poll}, }; use flume::{r#async::RecvFut, SendError, Sender}; +#[cfg(feature = "builtin-queue")] +use std::time::Duration; use tasks::message::CoreMessage; use tracing::instrument; @@ -54,8 +62,13 @@ pub struct Driver { config: Config, self_mute: bool, sender: Sender, + // Making this an Option is an abhorrent hack to coerce the borrow checker + // into letting us have an &TrackQueue at the same time as an &mut Driver. + // This is probably preferable to cloning the driver: Arc<...> should be nonzero + // and if the compiler's smart we'll just codegen a pointer swap. It definitely makes + // use of NonZero. #[cfg(feature = "builtin-queue")] - queue: TrackQueue, + queue: Option, } impl Driver { @@ -63,6 +76,7 @@ impl Driver { /// /// This will create the core voice tasks in the background. #[inline] + #[must_use] pub fn new(config: Config) -> Self { let sender = Self::start_inner(config.clone()); @@ -71,7 +85,7 @@ impl Driver { self_mute: false, sender, #[cfg(feature = "builtin-queue")] - queue: Default::default(), + queue: Some(TrackQueue::default()), } } @@ -136,63 +150,45 @@ impl Driver { self.self_mute } - /// Plays audio from a source, returning a handle for further control. - /// - /// This can be a source created via [`ffmpeg`] or [`ytdl`]. - /// - /// [`ffmpeg`]: crate::input::ffmpeg - /// [`ytdl`]: crate::input::ytdl - #[instrument(skip(self))] - pub fn play_source(&mut self, source: Input) -> TrackHandle { - let (player, handle) = super::create_player(source); - self.send(CoreMessage::AddTrack(player)); - - handle + /// Plays audio from an input, returning a handle for further control. + #[instrument(skip(self, input))] + pub fn play_input(&mut self, input: Input) -> TrackHandle { + self.play(input.into()) } - /// Plays audio from a source, returning a handle for further control. + /// Plays audio from an input, returning a handle for further control. /// - /// Unlike [`play_source`], this stops all other sources attached + /// Unlike [`Self::play_input`], this stops all other inputs attached /// to the channel. - /// - /// [`play_source`]: Driver::play_source - #[instrument(skip(self))] - pub fn play_only_source(&mut self, source: Input) -> TrackHandle { - let (player, handle) = super::create_player(source); - self.send(CoreMessage::SetTrack(Some(player))); - - handle + #[instrument(skip(self, input))] + pub fn play_only_input(&mut self, input: Input) -> TrackHandle { + self.play_only(input.into()) } /// Plays audio from a [`Track`] object. /// - /// This will be one half of the return value of [`create_player`]. - /// The main difference between this function and [`play_source`] is + /// The main difference between this function and [`Self::play_input`] is /// that this allows for direct manipulation of the [`Track`] object /// before it is passed over to the voice and mixing contexts. - /// - /// [`create_player`]: crate::tracks::create_player - /// [`create_player`]: crate::tracks::Track - /// [`play_source`]: Driver::play_source - #[instrument(skip(self))] - pub fn play(&mut self, track: Track) { - self.send(CoreMessage::AddTrack(track)); + #[instrument(skip(self, track))] + pub fn play(&mut self, track: Track) -> TrackHandle { + let (handle, ctx) = track.into_context(); + self.send(CoreMessage::AddTrack(ctx)); + + handle } /// Exclusively plays audio from a [`Track`] object. /// - /// This will be one half of the return value of [`create_player`]. - /// As in [`play_only_source`], this stops all other sources attached to the - /// channel. Like [`play`], however, this allows for direct manipulation of the + /// As in [`Self::play_only_input`], this stops all other sources attached to the + /// channel. Like [`Self::play`], however, this allows for direct manipulation of the /// [`Track`] object before it is passed over to the voice and mixing contexts. - /// - /// [`create_player`]: crate::tracks::create_player - /// [`Track`]: crate::tracks::Track - /// [`play_only_source`]: Driver::play_only_source - /// [`play`]: Driver::play - #[instrument(skip(self))] - pub fn play_only(&mut self, track: Track) { - self.send(CoreMessage::SetTrack(Some(track))); + #[instrument(skip(self, track))] + pub fn play_only(&mut self, track: Track) -> TrackHandle { + let (handle, ctx) = track.into_context(); + self.send(CoreMessage::SetTrack(Some(ctx))); + + handle } /// Sets the bitrate for encoding Opus packets sent along @@ -204,20 +200,20 @@ impl Driver { /// Alternatively, `Auto` and `Max` remain available. #[instrument(skip(self))] pub fn set_bitrate(&mut self, bitrate: Bitrate) { - self.send(CoreMessage::SetBitrate(bitrate)) + self.send(CoreMessage::SetBitrate(bitrate)); } /// Stops playing audio from all sources, if any are set. #[instrument(skip(self))] pub fn stop(&mut self) { - self.send(CoreMessage::SetTrack(None)) + self.send(CoreMessage::SetTrack(None)); } /// Sets the configuration for this driver (and parent `Call`, if applicable). #[instrument(skip(self))] pub fn set_config(&mut self, config: Config) { self.config = config.clone(); - self.send(CoreMessage::SetConfig(config)) + self.send(CoreMessage::SetConfig(config)); } /// Returns a view of this driver's configuration. @@ -237,7 +233,6 @@ impl Driver { /// within the supplied function or closure. *Taking excess time could prevent /// timely sending of packets, causing audio glitches and delays*. /// - /// [`Track`]: crate::tracks::Track /// [`TrackEvent`]: crate::events::TrackEvent /// [`EventContext`]: crate::events::EventContext #[instrument(skip(self, action))] @@ -267,41 +262,53 @@ impl Driver { /// Returns a reference to this driver's built-in queue. /// /// Requires the `"builtin-queue"` feature. - /// Queue additions should be made via [`enqueue`] and - /// [`enqueue_source`]. - /// - /// [`enqueue`]: Driver::enqueue - /// [`enqueue_source`]: Driver::enqueue_source + /// Queue additions should be made via [`Driver::enqueue`] and + /// [`Driver::enqueue_input`]. + #[must_use] pub fn queue(&self) -> &TrackQueue { - &self.queue + self.queue + .as_ref() + .expect("Queue: The only case this can fail is if a previous queue operation panicked.") } /// Adds an audio [`Input`] to this driver's built-in queue. /// /// Requires the `"builtin-queue"` feature. - /// - /// [`Input`]: crate::input::Input - pub fn enqueue_source(&mut self, source: Input) -> TrackHandle { - let (track, handle) = tracks::create_player(source); - self.enqueue(track); - - handle + pub async fn enqueue_input(&mut self, input: Input) -> TrackHandle { + self.enqueue(input.into()).await } /// Adds an existing [`Track`] to this driver's built-in queue. /// /// Requires the `"builtin-queue"` feature. + pub async fn enqueue(&mut self, mut track: Track) -> TrackHandle { + let preload_time = TrackQueue::get_preload_time(&mut track).await; + self.enqueue_with_preload(track, preload_time) + } + + /// Add an existing [`Track`] to the queue, using a known time to preload the next track. /// - /// [`Track`]: crate::tracks::Track - pub fn enqueue(&mut self, mut track: Track) { - self.queue.add_raw(&mut track); - self.play(track); + /// See [`TrackQueue::add_with_preload`] for how `preload_time` is used. + /// + /// Requires the `"builtin-queue"` feature. + pub fn enqueue_with_preload( + &mut self, + track: Track, + preload_time: Option, + ) -> TrackHandle { + let queue = self.queue.take().expect( + "Enqueue: The only case this can fail is if a previous queue operation panicked.", + ); + let handle = queue.add_with_preload(track, self, preload_time); + self.queue = Some(queue); + + handle } } impl Default for Driver { fn default() -> Self { - Self::new(Default::default()) + Self::new(Config::default()) } } @@ -309,7 +316,7 @@ impl Drop for Driver { /// Leaves the current connected voice channel, if connected to one, and /// forgets all configurations relevant to this Handler. fn drop(&mut self) { - let _ = self.sender.send(CoreMessage::Poison); + drop(self.sender.send(CoreMessage::Poison)); } } @@ -317,8 +324,6 @@ impl Drop for Driver { /// /// This future awaits the *result* of a connection; the driver /// is messaged at the time of the call. -/// -/// [`Driver::connect`]: Driver::connect pub struct Connect { inner: RecvFut<'static, Result<()>>, } diff --git a/src/driver/retry/mod.rs b/src/driver/retry/mod.rs index e25374b..bd7e8a6 100644 --- a/src/driver/retry/mod.rs +++ b/src/driver/retry/mod.rs @@ -28,7 +28,7 @@ pub struct Retry { impl Default for Retry { fn default() -> Self { Self { - strategy: Strategy::Backoff(Default::default()), + strategy: Strategy::Backoff(ExponentialBackoff::default()), retry_limit: Some(5), } } @@ -40,7 +40,7 @@ impl Retry { last_wait: Option, attempts: usize, ) -> Option { - if self.retry_limit.map(|a| attempts < a).unwrap_or(true) { + if self.retry_limit.map_or(true, |a| attempts < a) { Some(self.strategy.retry_in(last_wait)) } else { None diff --git a/src/driver/retry/strategy.rs b/src/driver/retry/strategy.rs index 6de58e7..4501728 100644 --- a/src/driver/retry/strategy.rs +++ b/src/driver/retry/strategy.rs @@ -58,7 +58,7 @@ impl Default for ExponentialBackoff { impl ExponentialBackoff { pub(crate) fn retry_in(&self, last_wait: Option) -> Duration { - let attempt = last_wait.map(|t| 2 * t).unwrap_or(self.min); + let attempt = last_wait.map_or(self.min, |t| 2 * t); let perturb = (1.0 - (self.jitter * 2.0 * (random::() - 1.0))) .max(0.0) .min(2.0); diff --git a/src/driver/tasks/disposal.rs b/src/driver/tasks/disposal.rs index b10a56f..0b13014 100644 --- a/src/driver/tasks/disposal.rs +++ b/src/driver/tasks/disposal.rs @@ -9,10 +9,5 @@ use tracing::instrument; /// to prevent deadline misses. #[instrument(skip(mix_rx))] pub(crate) fn runner(mix_rx: Receiver) { - loop { - match mix_rx.recv() { - Err(_) | Ok(DisposalMessage::Poison) => break, - _ => {}, - } - } + while mix_rx.recv().is_ok() {} } diff --git a/src/driver/tasks/error.rs b/src/driver/tasks/error.rs index c56319b..10a1bb7 100644 --- a/src/driver/tasks/error.rs +++ b/src/driver/tasks/error.rs @@ -32,9 +32,7 @@ impl Error { pub(crate) fn should_trigger_connect(&self) -> bool { matches!( self, - Error::InterconnectFailure(Recipient::AuxNetwork) - | Error::InterconnectFailure(Recipient::UdpRx) - | Error::InterconnectFailure(Recipient::UdpTx) + Error::InterconnectFailure(Recipient::AuxNetwork | Recipient::UdpRx | Recipient::UdpTx) ) } diff --git a/src/driver/tasks/events.rs b/src/driver/tasks/events.rs index 072f2fb..e85f0d6 100644 --- a/src/driver/tasks/events.rs +++ b/src/driver/tasks/events.rs @@ -1,7 +1,7 @@ use super::message::*; use crate::{ events::{EventStore, GlobalEvents, TrackEvent}, - tracks::{TrackHandle, TrackState}, + tracks::{ReadyState, TrackHandle, TrackState}, }; use flume::Receiver; use tracing::{debug, info, instrument, trace}; @@ -14,14 +14,13 @@ pub(crate) async fn runner(_interconnect: Interconnect, evt_rx: Receiver = vec![]; let mut handles: Vec = vec![]; - loop { - use EventMessage::*; - match evt_rx.recv_async().await { - Ok(AddGlobalEvent(data)) => { + while let Ok(msg) = evt_rx.recv_async().await { + match msg { + EventMessage::AddGlobalEvent(data) => { info!("Global event added."); global.add_event(data); }, - Ok(AddTrackEvent(i, data)) => { + EventMessage::AddTrackEvent(i, data) => { info!("Adding event to track {}.", i); let event_store = events @@ -33,7 +32,7 @@ pub(crate) async fn runner(_interconnect: Interconnect, evt_rx: Receiver { + EventMessage::FireCoreEvent(ctx) => { let ctx = ctx.to_user_context(); let evt = ctx .to_core_event() @@ -43,19 +42,17 @@ pub(crate) async fn runner(_interconnect: Interconnect, evt_rx: Receiver { + EventMessage::RemoveGlobalEvents => { global.remove_handlers(); }, - Ok(AddTrack(store, state, handle)) => { + EventMessage::AddTrack(store, state, handle) => { events.push(store); states.push(state); handles.push(handle); info!("Event state for track {} added", events.len()); }, - Ok(ChangeState(i, change)) => { - use TrackStateChange::*; - + EventMessage::ChangeState(i, change) => { let max_states = states.len(); debug!( "Changing state for track {} of {}: {:?}", @@ -67,53 +64,74 @@ pub(crate) async fn runner(_interconnect: Interconnect, evt_rx: Receiver { - let old = state.playing; - state.playing = mode; - if old != mode { - global.fire_track_event(mode.as_track_event(), i); + TrackStateChange::Mode(mut mode) => { + std::mem::swap(&mut state.playing, &mut mode); + if state.playing != mode { + global.fire_track_event(state.playing.as_track_event(), i); + if let Some(other_evts) = state.playing.also_fired_track_events() { + for evt in other_evts { + global.fire_track_event(evt, i); + } + } } }, - Volume(vol) => { + TrackStateChange::Volume(vol) => { state.volume = vol; }, - Position(pos) => { + TrackStateChange::Position(pos) => { // Currently, only Tick should fire time events. state.position = pos; }, - Loops(loops, user_set) => { + TrackStateChange::Loops(loops, user_set) => { state.loops = loops; if !user_set { global.fire_track_event(TrackEvent::Loop, i); } }, - Total(new) => { + TrackStateChange::Total(new) => { // Massive, unprecedented state changes. *state = new; }, + TrackStateChange::Ready(ready_state) => { + state.ready = ready_state; + + match ready_state { + ReadyState::Playable => { + global.fire_track_event(TrackEvent::Playable, i); + }, + ReadyState::Preparing => { + global.fire_track_event(TrackEvent::Preparing, i); + }, + ReadyState::Uninitialised => {}, + } + }, } }, - Ok(RemoveTrack(i)) => { - info!("Event state for track {} of {} removed.", i, events.len()); - - events.swap_remove(i); - states.swap_remove(i); - handles.swap_remove(i); - }, - Ok(RemoveAllTracks) => { + EventMessage::RemoveAllTracks => { info!("Event state for all tracks removed."); events.clear(); states.clear(); handles.clear(); }, - Ok(Tick) => { + EventMessage::Tick => { // NOTE: this should fire saved up blocks of state change evts. global.tick(&mut events, &mut states, &mut handles).await; + + let mut i = 0; + while i < states.len() { + if states[i].playing.is_done() { + info!("Event state for track {} of {} removed.", i, events.len()); + + events.swap_remove(i); + states.swap_remove(i); + handles.swap_remove(i); + } else { + i += 1; + } + } }, - Err(_) | Ok(Poison) => { - break; - }, + EventMessage::Poison => break, } } diff --git a/src/driver/tasks/message/core.rs b/src/driver/tasks/message/core.rs index 5fbc0df..b614526 100644 --- a/src/driver/tasks/message/core.rs +++ b/src/driver/tasks/message/core.rs @@ -3,20 +3,18 @@ use crate::{ driver::{connection::error::Error, Bitrate, Config}, events::{context_data::DisconnectReason, EventData}, - tracks::Track, + tracks::{Track, TrackCommand, TrackHandle}, ConnectionInfo, }; -use flume::Sender; +use flume::{Receiver, Sender}; -#[allow(clippy::large_enum_variant)] -#[derive(Debug)] pub enum CoreMessage { ConnectWithResult(ConnectionInfo, Sender>), RetryConnect(usize), SignalWsClosure(usize, ConnectionInfo, Option), Disconnect, - SetTrack(Option), - AddTrack(Track), + SetTrack(Option), + AddTrack(TrackContext), SetBitrate(Bitrate), AddEvent(EventData), RemoveGlobalEvents, @@ -27,3 +25,9 @@ pub enum CoreMessage { RebuildInterconnect, Poison, } + +pub struct TrackContext { + pub track: Track, + pub handle: TrackHandle, + pub receiver: Receiver, +} diff --git a/src/driver/tasks/message/disposal.rs b/src/driver/tasks/message/disposal.rs index 84df7e6..795e5bc 100644 --- a/src/driver/tasks/message/disposal.rs +++ b/src/driver/tasks/message/disposal.rs @@ -1,9 +1,8 @@ #![allow(missing_docs)] -use crate::tracks::Track; +use crate::{driver::tasks::mixer::InternalTrack, tracks::TrackHandle}; pub enum DisposalMessage { - Track(Track), - - Poison, + Track(Box), + Handle(TrackHandle), } diff --git a/src/driver/tasks/message/events.rs b/src/driver/tasks/message/events.rs index 2125fd8..9e464da 100644 --- a/src/driver/tasks/message/events.rs +++ b/src/driver/tasks/message/events.rs @@ -2,7 +2,7 @@ use crate::{ events::{CoreContext, EventData, EventStore}, - tracks::{LoopState, PlayMode, TrackHandle, TrackState}, + tracks::{LoopState, PlayMode, ReadyState, TrackHandle, TrackState}, }; use std::time::Duration; @@ -16,7 +16,6 @@ pub enum EventMessage { AddTrack(EventStore, TrackState, TrackHandle), ChangeState(usize, TrackStateChange), - RemoveTrack(usize), RemoveAllTracks, Tick, @@ -31,4 +30,5 @@ pub enum TrackStateChange { // Bool indicates user-set. Loops(LoopState, bool), Total(TrackState), + Ready(ReadyState), } diff --git a/src/driver/tasks/message/mixer.rs b/src/driver/tasks/message/mixer.rs index 220c26d..175295a 100644 --- a/src/driver/tasks/message/mixer.rs +++ b/src/driver/tasks/message/mixer.rs @@ -1,12 +1,14 @@ #![allow(missing_docs)] -use super::{Interconnect, UdpRxMessage, UdpTxMessage, WsMessage}; +use super::{Interconnect, TrackContext, UdpRxMessage, UdpTxMessage, WsMessage}; use crate::{ driver::{Bitrate, Config, CryptoState}, - tracks::Track, + input::{AudioStreamError, Compose, Parsed}, }; use flume::Sender; +use std::sync::Arc; +use symphonia_core::{errors::Error as SymphoniaError, formats::SeekedTo}; use xsalsa20poly1305::XSalsa20Poly1305 as Cipher; pub struct MixerConnection { @@ -16,16 +18,9 @@ pub struct MixerConnection { pub udp_tx: Sender, } -impl Drop for MixerConnection { - fn drop(&mut self) { - let _ = self.udp_rx.send(UdpRxMessage::Poison); - let _ = self.udp_tx.send(UdpTxMessage::Poison); - } -} - pub enum MixerMessage { - AddTrack(Track), - SetTrack(Option), + AddTrack(TrackContext), + SetTrack(Option), SetBitrate(Bitrate), SetConfig(Config), @@ -40,3 +35,14 @@ pub enum MixerMessage { Poison, } + +pub enum MixerInputResultMessage { + CreateErr(Arc), + ParseErr(Arc), + Seek( + Parsed, + Option>, + Result>, + ), + Built(Parsed, Option>), +} diff --git a/src/driver/tasks/message/mod.rs b/src/driver/tasks/message/mod.rs index 54b27ba..0009c17 100644 --- a/src/driver/tasks/message/mod.rs +++ b/src/driver/tasks/message/mod.rs @@ -23,11 +23,11 @@ pub struct Interconnect { impl Interconnect { pub fn poison(&self) { - let _ = self.events.send(EventMessage::Poison); + drop(self.events.send(EventMessage::Poison)); } pub fn poison_all(&self) { - let _ = self.mixer.send(MixerMessage::Poison); + drop(self.mixer.send(MixerMessage::Poison)); self.poison(); } @@ -46,8 +46,9 @@ impl Interconnect { }); // Make mixer aware of new targets... - let _ = self - .mixer - .send(MixerMessage::ReplaceInterconnect(self.clone())); + drop( + self.mixer + .send(MixerMessage::ReplaceInterconnect(self.clone())), + ); } } diff --git a/src/driver/tasks/message/udp_rx.rs b/src/driver/tasks/message/udp_rx.rs index 9034090..12a6d0c 100644 --- a/src/driver/tasks/message/udp_rx.rs +++ b/src/driver/tasks/message/udp_rx.rs @@ -6,6 +6,4 @@ use crate::driver::Config; pub enum UdpRxMessage { SetConfig(Config), ReplaceInterconnect(Interconnect), - - Poison, } diff --git a/src/driver/tasks/message/udp_tx.rs b/src/driver/tasks/message/udp_tx.rs index d3dbf36..16b7ad1 100644 --- a/src/driver/tasks/message/udp_tx.rs +++ b/src/driver/tasks/message/udp_tx.rs @@ -1,6 +1,4 @@ #![allow(missing_docs)] -pub enum UdpTxMessage { - Packet(Vec), // TODO: do something cheaper. - Poison, -} +// TODO: do something cheaper. +pub type UdpTxMessage = Vec; diff --git a/src/driver/tasks/message/ws.rs b/src/driver/tasks/message/ws.rs index 1cd7e49..4faf683 100644 --- a/src/driver/tasks/message/ws.rs +++ b/src/driver/tasks/message/ws.rs @@ -3,12 +3,9 @@ use super::Interconnect; use crate::ws::WsStream; -#[allow(dead_code)] pub enum WsMessage { Ws(Box), ReplaceInterconnect(Interconnect), SetKeepalive(f64), Speaking(bool), - - Poison, } diff --git a/src/driver/tasks/mixer.rs b/src/driver/tasks/mixer.rs deleted file mode 100644 index 47653df..0000000 --- a/src/driver/tasks/mixer.rs +++ /dev/null @@ -1,628 +0,0 @@ -use super::{disposal, error::Result, message::*}; -use crate::{ - constants::*, - tracks::{PlayMode, Track}, - Config, -}; -use audiopus::{ - coder::Encoder as OpusEncoder, - softclip::SoftClip, - Application as CodingMode, - Bitrate, - Channels, -}; -use discortp::{ - rtp::{MutableRtpPacket, RtpPacket}, - MutablePacket, -}; -use flume::{Receiver, Sender, TryRecvError}; -use rand::random; -use std::{convert::TryInto, time::Instant}; -use tokio::runtime::Handle; -use tracing::{debug, error, instrument}; -use xsalsa20poly1305::TAG_SIZE; - -pub struct Mixer { - pub async_handle: Handle, - pub bitrate: Bitrate, - pub config: Config, - pub conn_active: Option, - pub deadline: Instant, - pub disposer: Sender, - pub encoder: OpusEncoder, - pub interconnect: Interconnect, - pub mix_rx: Receiver, - pub muted: bool, - pub packet: [u8; VOICE_PACKET_MAX], - pub prevent_events: bool, - pub silence_frames: u8, - pub skip_sleep: bool, - pub soft_clip: SoftClip, - pub tracks: Vec, - pub ws: Option>, -} - -fn new_encoder(bitrate: Bitrate) -> Result { - let mut encoder = OpusEncoder::new(SAMPLE_RATE, Channels::Stereo, CodingMode::Audio)?; - encoder.set_bitrate(bitrate)?; - - Ok(encoder) -} - -impl Mixer { - pub fn new( - mix_rx: Receiver, - async_handle: Handle, - interconnect: Interconnect, - config: Config, - ) -> Self { - let bitrate = DEFAULT_BITRATE; - let encoder = new_encoder(bitrate) - .expect("Failed to create encoder in mixing thread with known-good values."); - let soft_clip = SoftClip::new(Channels::Stereo); - - let mut packet = [0u8; VOICE_PACKET_MAX]; - - let mut rtp = MutableRtpPacket::new(&mut packet[..]).expect( - "FATAL: Too few bytes in self.packet for RTP header.\ - (Blame: VOICE_PACKET_MAX?)", - ); - rtp.set_version(RTP_VERSION); - rtp.set_payload_type(RTP_PROFILE_TYPE); - rtp.set_sequence(random::().into()); - rtp.set_timestamp(random::().into()); - - let tracks = Vec::with_capacity(1.max(config.preallocated_tracks)); - - // Create an object disposal thread here. - let (disposer, disposal_rx) = flume::unbounded(); - std::thread::spawn(move || disposal::runner(disposal_rx)); - - Self { - async_handle, - bitrate, - config, - conn_active: None, - deadline: Instant::now(), - disposer, - encoder, - interconnect, - mix_rx, - muted: false, - packet, - prevent_events: false, - silence_frames: 0, - skip_sleep: false, - soft_clip, - tracks, - ws: None, - } - } - - fn run(&mut self) { - let mut events_failure = false; - let mut conn_failure = false; - - 'runner: loop { - if self.conn_active.is_some() { - loop { - match self.mix_rx.try_recv() { - Ok(m) => { - let (events, conn, should_exit) = self.handle_message(m); - events_failure |= events; - conn_failure |= conn; - - if should_exit { - break 'runner; - } - }, - - Err(TryRecvError::Disconnected) => { - break 'runner; - }, - - Err(TryRecvError::Empty) => { - break; - }, - }; - } - - // The above action may have invalidated the connection; need to re-check! - if self.conn_active.is_some() { - if let Err(e) = self.cycle().and_then(|_| self.audio_commands_events()) { - events_failure |= e.should_trigger_interconnect_rebuild(); - conn_failure |= e.should_trigger_connect(); - - debug!("Mixer thread cycle: {:?}", e); - } - } - } else { - match self.mix_rx.recv() { - Ok(m) => { - let (events, conn, should_exit) = self.handle_message(m); - events_failure |= events; - conn_failure |= conn; - - if should_exit { - break 'runner; - } - }, - Err(_) => { - break 'runner; - }, - } - } - - // event failure? rebuild interconnect. - // ws or udp failure? full connect - // (soft reconnect is covered by the ws task.) - // - // in both cases, send failure is fatal, - // but will only occur on disconnect. - // expecting this is fairly noisy, so exit silently. - if events_failure { - self.prevent_events = true; - let sent = self - .interconnect - .core - .send(CoreMessage::RebuildInterconnect); - events_failure = false; - - if sent.is_err() { - break; - } - } - - if conn_failure { - self.conn_active = None; - let sent = self.interconnect.core.send(CoreMessage::FullReconnect); - conn_failure = false; - - if sent.is_err() { - break; - } - } - } - } - - #[inline] - fn handle_message(&mut self, msg: MixerMessage) -> (bool, bool, bool) { - let mut events_failure = false; - let mut conn_failure = false; - let mut should_exit = false; - - use MixerMessage::*; - - let error = match msg { - AddTrack(mut t) => { - t.source.prep_with_handle(self.async_handle.clone()); - self.add_track(t) - }, - SetTrack(t) => { - self.tracks.clear(); - - let mut out = self.fire_event(EventMessage::RemoveAllTracks); - - if let Some(mut t) = t { - t.source.prep_with_handle(self.async_handle.clone()); - - // Do this unconditionally: this affects local state infallibly, - // with the event installation being the remote part. - if let Err(e) = self.add_track(t) { - out = Err(e); - } - } - - out - }, - SetBitrate(b) => { - self.bitrate = b; - if let Err(e) = self.set_bitrate(b) { - error!("Failed to update bitrate {:?}", e); - } - Ok(()) - }, - SetMute(m) => { - self.muted = m; - Ok(()) - }, - SetConn(conn, ssrc) => { - self.conn_active = Some(conn); - let mut rtp = MutableRtpPacket::new(&mut self.packet[..]).expect( - "Too few bytes in self.packet for RTP header.\ - (Blame: VOICE_PACKET_MAX?)", - ); - rtp.set_ssrc(ssrc); - rtp.set_sequence(random::().into()); - rtp.set_timestamp(random::().into()); - self.deadline = Instant::now(); - Ok(()) - }, - DropConn => { - self.conn_active = None; - Ok(()) - }, - ReplaceInterconnect(i) => { - self.prevent_events = false; - if let Some(ws) = &self.ws { - conn_failure |= ws.send(WsMessage::ReplaceInterconnect(i.clone())).is_err(); - } - if let Some(conn) = &self.conn_active { - conn_failure |= conn - .udp_rx - .send(UdpRxMessage::ReplaceInterconnect(i.clone())) - .is_err(); - } - self.interconnect = i; - - self.rebuild_tracks() - }, - SetConfig(new_config) => { - self.config = new_config.clone(); - - if self.tracks.capacity() < self.config.preallocated_tracks { - self.tracks - .reserve(self.config.preallocated_tracks - self.tracks.len()); - } - - if let Some(conn) = &self.conn_active { - conn_failure |= conn - .udp_rx - .send(UdpRxMessage::SetConfig(new_config)) - .is_err(); - } - - Ok(()) - }, - RebuildEncoder => match new_encoder(self.bitrate) { - Ok(encoder) => { - self.encoder = encoder; - Ok(()) - }, - Err(e) => { - error!("Failed to rebuild encoder. Resetting bitrate. {:?}", e); - self.bitrate = DEFAULT_BITRATE; - self.encoder = new_encoder(self.bitrate) - .expect("Failed fallback rebuild of OpusEncoder with safe inputs."); - Ok(()) - }, - }, - Ws(new_ws_handle) => { - self.ws = new_ws_handle; - Ok(()) - }, - Poison => { - should_exit = true; - Ok(()) - }, - }; - - if let Err(e) = error { - events_failure |= e.should_trigger_interconnect_rebuild(); - conn_failure |= e.should_trigger_connect(); - } - - (events_failure, conn_failure, should_exit) - } - - #[inline] - fn fire_event(&self, event: EventMessage) -> Result<()> { - // As this task is responsible for noticing the potential death of an event context, - // it's responsible for not forcibly recreating said context repeatedly. - if !self.prevent_events { - self.interconnect.events.send(event)?; - Ok(()) - } else { - Ok(()) - } - } - - #[inline] - fn add_track(&mut self, mut track: Track) -> Result<()> { - let evts = track.events.take().unwrap_or_default(); - let state = track.state(); - let handle = track.handle.clone(); - - self.tracks.push(track); - - self.interconnect - .events - .send(EventMessage::AddTrack(evts, state, handle))?; - - Ok(()) - } - - // rebuilds the event thread's view of each track, in event of a full rebuild. - #[inline] - fn rebuild_tracks(&mut self) -> Result<()> { - for track in self.tracks.iter_mut() { - let evts = track.events.take().unwrap_or_default(); - let state = track.state(); - let handle = track.handle.clone(); - - self.interconnect - .events - .send(EventMessage::AddTrack(evts, state, handle))?; - } - - Ok(()) - } - - #[inline] - fn audio_commands_events(&mut self) -> Result<()> { - // Apply user commands. - for (i, track) in self.tracks.iter_mut().enumerate() { - // This causes fallible event system changes, - // but if the event thread has died then we'll certainly - // detect that on the tick later. - // Changes to play state etc. MUST all be handled. - track.process_commands(i, &self.interconnect); - } - - // TODO: do without vec? - let mut i = 0; - let mut to_remove = Vec::with_capacity(self.tracks.len()); - while i < self.tracks.len() { - let track = self - .tracks - .get_mut(i) - .expect("Tried to remove an illegal track index."); - - if track.playing.is_done() { - let p_state = track.playing(); - let to_drop = self.tracks.swap_remove(i); - to_remove.push(i); - self.fire_event(EventMessage::ChangeState( - i, - TrackStateChange::Mode(p_state), - ))?; - let _ = self.disposer.send(DisposalMessage::Track(to_drop)); - } else { - i += 1; - } - } - - // Tick - self.fire_event(EventMessage::Tick)?; - - // Then do removals. - for i in &to_remove[..] { - self.fire_event(EventMessage::RemoveTrack(*i))?; - } - - Ok(()) - } - - #[inline] - fn march_deadline(&mut self) { - if self.skip_sleep { - return; - } - - std::thread::sleep(self.deadline.saturating_duration_since(Instant::now())); - self.deadline += TIMESTEP_LENGTH; - } - - pub fn cycle(&mut self) -> Result<()> { - let mut mix_buffer = [0f32; STEREO_FRAME_SIZE]; - - // Walk over all the audio files, combining into one audio frame according - // to volume, play state, etc. - let mut mix_len = { - let mut rtp = MutableRtpPacket::new(&mut self.packet[..]).expect( - "FATAL: Too few bytes in self.packet for RTP header.\ - (Blame: VOICE_PACKET_MAX?)", - ); - - let payload = rtp.payload_mut(); - - // self.mix_tracks(&mut payload[TAG_SIZE..], &mut mix_buffer) - mix_tracks( - &mut payload[TAG_SIZE..], - &mut mix_buffer, - &mut self.tracks, - &self.interconnect, - self.prevent_events, - ) - }; - - self.soft_clip.apply((&mut mix_buffer[..]).try_into()?)?; - - if self.muted { - mix_len = MixType::MixedPcm(0); - } - - if mix_len == MixType::MixedPcm(0) { - if self.silence_frames > 0 { - self.silence_frames -= 1; - - // Explicit "Silence" frame. - let mut rtp = MutableRtpPacket::new(&mut self.packet[..]).expect( - "FATAL: Too few bytes in self.packet for RTP header.\ - (Blame: VOICE_PACKET_MAX?)", - ); - - let payload = rtp.payload_mut(); - - (&mut payload[TAG_SIZE..TAG_SIZE + SILENT_FRAME.len()]) - .copy_from_slice(&SILENT_FRAME[..]); - - mix_len = MixType::Passthrough(SILENT_FRAME.len()); - } else { - // Per official guidelines, send 5x silence BEFORE we stop speaking. - if let Some(ws) = &self.ws { - // NOTE: this should prevent a catastrophic thread pileup. - // A full reconnect might cause an inner closed connection. - // It's safer to leave the central task to clean this up and - // pass the mixer a new channel. - let _ = ws.send(WsMessage::Speaking(false)); - } - - self.march_deadline(); - - return Ok(()); - } - } else { - self.silence_frames = 5; - } - - if let Some(ws) = &self.ws { - ws.send(WsMessage::Speaking(true))?; - } - - self.march_deadline(); - self.prep_and_send_packet(mix_buffer, mix_len)?; - - Ok(()) - } - - fn set_bitrate(&mut self, bitrate: Bitrate) -> Result<()> { - self.encoder.set_bitrate(bitrate).map_err(Into::into) - } - - #[inline] - fn prep_and_send_packet(&mut self, buffer: [f32; 1920], mix_len: MixType) -> Result<()> { - let conn = self - .conn_active - .as_mut() - .expect("Shouldn't be mixing packets without access to a cipher + UDP dest."); - - let index = { - let mut rtp = MutableRtpPacket::new(&mut self.packet[..]).expect( - "FATAL: Too few bytes in self.packet for RTP header.\ - (Blame: VOICE_PACKET_MAX?)", - ); - - let payload = rtp.payload_mut(); - let crypto_mode = conn.crypto_state.kind(); - - let payload_len = match mix_len { - MixType::Passthrough(opus_len) => opus_len, - MixType::MixedPcm(_samples) => { - let total_payload_space = payload.len() - crypto_mode.payload_suffix_len(); - self.encoder.encode_float( - &buffer[..STEREO_FRAME_SIZE], - &mut payload[TAG_SIZE..total_payload_space], - )? - }, - }; - - let final_payload_size = conn - .crypto_state - .write_packet_nonce(&mut rtp, TAG_SIZE + payload_len); - - conn.crypto_state.kind().encrypt_in_place( - &mut rtp, - &conn.cipher, - final_payload_size, - )?; - - RtpPacket::minimum_packet_size() + final_payload_size - }; - - // TODO: This is dog slow, don't do this. - // Can we replace this with a shared ring buffer + semaphore? - // i.e., do something like double/triple buffering in graphics. - conn.udp_tx - .send(UdpTxMessage::Packet(self.packet[..index].to_vec()))?; - - let mut rtp = MutableRtpPacket::new(&mut self.packet[..]).expect( - "FATAL: Too few bytes in self.packet for RTP header.\ - (Blame: VOICE_PACKET_MAX?)", - ); - rtp.set_sequence(rtp.get_sequence() + 1); - rtp.set_timestamp(rtp.get_timestamp() + MONO_FRAME_SIZE as u32); - - Ok(()) - } -} - -#[derive(Debug, Eq, PartialEq)] -enum MixType { - Passthrough(usize), - MixedPcm(usize), -} - -#[inline] -fn mix_tracks<'a>( - opus_frame: &'a mut [u8], - mix_buffer: &mut [f32; STEREO_FRAME_SIZE], - tracks: &mut Vec, - interconnect: &Interconnect, - prevent_events: bool, -) -> MixType { - let mut len = 0; - - // Opus frame passthrough. - // This requires that we have only one track, who has volume 1.0, and an - // Opus codec type. - let do_passthrough = tracks.len() == 1 && { - let track = &tracks[0]; - (track.volume - 1.0).abs() < f32::EPSILON && track.source.supports_passthrough() - }; - - for (i, track) in tracks.iter_mut().enumerate() { - let vol = track.volume; - let stream = &mut track.source; - - if track.playing != PlayMode::Play { - continue; - } - - let (temp_len, opus_len) = if do_passthrough { - (0, track.source.read_opus_frame(opus_frame).ok()) - } else { - (stream.mix(mix_buffer, vol), None) - }; - - len = len.max(temp_len); - if temp_len > 0 || opus_len.is_some() { - track.step_frame(); - } else if track.do_loop() { - if let Ok(time) = track.seek_time(Default::default()) { - // have to reproduce self.fire_event here - // to circumvent the borrow checker's lack of knowledge. - // - // In event of error, one of the later event calls will - // trigger the event thread rebuild: it is more prudent that - // the mixer works as normal right now. - if !prevent_events { - let _ = interconnect.events.send(EventMessage::ChangeState( - i, - TrackStateChange::Position(time), - )); - let _ = interconnect.events.send(EventMessage::ChangeState( - i, - TrackStateChange::Loops(track.loops, false), - )); - } - } - } else { - track.end(); - } - - if let Some(opus_len) = opus_len { - return MixType::Passthrough(opus_len); - } - } - - MixType::MixedPcm(len) -} - -/// The mixing thread is a synchronous context due to its compute-bound nature. -/// -/// We pass in an async handle for the benefit of some Input classes (e.g., restartables) -/// who need to run their restart code elsewhere and return blank data until such time. -#[instrument(skip(interconnect, mix_rx, async_handle))] -pub(crate) fn runner( - interconnect: Interconnect, - mix_rx: Receiver, - async_handle: Handle, - config: Config, -) { - let mut mixer = Mixer::new(mix_rx, async_handle, interconnect, config); - - mixer.run(); - - let _ = mixer.disposer.send(DisposalMessage::Poison); -} diff --git a/src/driver/tasks/mixer/mix_logic.rs b/src/driver/tasks/mixer/mix_logic.rs new file mode 100644 index 0000000..da0cb37 --- /dev/null +++ b/src/driver/tasks/mixer/mix_logic.rs @@ -0,0 +1,444 @@ +use super::*; + +/// Mix a track's audio stream into either the shared mixing buffer, or directly into the output +/// packet ("passthrough") when possible. +/// +/// Passthrough is highest performance, but the source MUST be opus, have 20ms frames, and be the only +/// live track. In this case we copy the opus-encoded data with no changes. Otherwise, we fall back to +/// below. +/// +/// There are a few functional requirements here for non-passthrough mixing that make it tricky: +/// * Input frame lengths are not congruent with what we need to send (i.e., 26.12ms in MP3 vs +/// needed 20ms). +/// * Input audio arrives at a different sample rate from required (i.e., 44.1 vs needed 48 kHz). +/// * Input data may not be `f32`s. +/// * Input data may not match stereo/mono of desired output. +/// +/// All of the above challenges often happen at once. The rough pipeline in processing is: +/// +/// until source end or 20 ms taken: +/// (use previous frame 'til empty / get new frame) -> [resample] -> [audio += vol * (sample as f32)] +/// +/// Typically, we mix between a subset of the input packet and the output buf because the 20ms window +/// straddles packet boundaries. If there's enough space AND 48kHz AND receive f32s, then we use a fast +/// path. +/// +/// In the mono -> stereo case, we duplicate across all target channels. In stereo -> mono, we average +/// the samples from each channel. +/// +/// To avoid needing to hold onto resampled data longer than one mix cycle, we take enough input samples +/// to fill a chunk of the mixer (e.g., 10ms == 20ms / 2) so that they will all be used. +/// +/// This is a fairly annoying piece of code to reason about, mainly because you need to hold so many +/// internal positions into: the mix buffer, resample buffers, and previous/current packets +/// for a stream. +#[inline] +pub fn mix_symph_indiv( + // shared buffer to mix into. + symph_mix: &mut AudioBuffer, + // buffer to hold built up packet + resample_scratch: &mut AudioBuffer, + // the input stream to use + input: &mut Parsed, + // resampler state and positions into partially read packets + local_state: &mut DecodeState, + // volume of this source + volume: f32, + // window into the output UDP buffer to copy opus frames into. + // This is set to `Some` IF passthrough is possible (i.e., one live source). + mut opus_slot: Option<&mut [u8]>, +) -> (MixType, MixStatus) { + let mut samples_written = 0; + let mut resample_in_progress = false; + let mut track_status = MixStatus::Live; + let codec_type = input.decoder.codec_params().codec; + + resample_scratch.clear(); + + while samples_written != MONO_FRAME_SIZE { + // fetch a packet: either in progress, passthrough (early exit), or + let source_packet = if local_state.inner_pos != 0 { + Some(input.decoder.last_decoded()) + } else if let Ok(pkt) = input.format.next_packet() { + if pkt.track_id() != input.track_id { + continue; + } + + let buf = pkt.buf(); + + // Opus packet passthrough special case. + if codec_type == CODEC_TYPE_OPUS && local_state.passthrough != Passthrough::Block { + if let Some(slot) = opus_slot.as_mut() { + let sample_ct = buf + .try_into() + .and_then(|buf| audiopus::packet::nb_samples(buf, SAMPLE_RATE)); + + // We don't actually block passthrough until a few violations are + // seen. The main one is that most Opus tracks end on a sub-20ms + // frame, particularly on Youtube. + // However, a frame that's bigger than the target buffer is an instant block. + let buf_size_fatal = buf.len() <= slot.len(); + + if match sample_ct { + Ok(MONO_FRAME_SIZE) => true, + _ => !local_state.record_and_check_passthrough_strike_final(buf_size_fatal), + } { + slot.write_all(buf) + .expect("Bounds check performed, and failure will block passthrough."); + + return (MixType::Passthrough(buf.len()), MixStatus::Live); + } + } + } + + input + .decoder + .decode(&pkt) + .map_err(|e| { + track_status = e.into(); + }) + .ok() + } else { + track_status = MixStatus::Ended; + None + }; + + // Cleanup: failed to get the next packet, but still have to convert and mix scratch. + if source_packet.is_none() { + if resample_in_progress { + // fill up remainder of buf with zeroes, resample, mix + let (chan_c, resampler, rs_out_buf) = local_state.resampler.as_mut().unwrap(); + let in_len = resample_scratch.frames(); + let to_render = resampler.input_frames_next().saturating_sub(in_len); + + if to_render != 0 { + resample_scratch.render_reserved(Some(to_render)); + for plane in resample_scratch.planes_mut().planes() { + for val in &mut plane[in_len..] { + *val = 0.0f32; + } + } + } + + // Luckily, we make use of the WHOLE input buffer here. + resampler + .process_into_buffer( + &resample_scratch.planes().planes()[..*chan_c], + rs_out_buf, + None, + ) + .unwrap(); + + // Calculate true end position using sample rate math + let ratio = (rs_out_buf[0].len() as f32) / (resample_scratch.frames() as f32); + let out_samples = (ratio * (in_len as f32)).round() as usize; + + mix_resampled(rs_out_buf, symph_mix, samples_written, volume); + + samples_written += out_samples; + } + + break; + } + + let source_packet = source_packet.unwrap(); + + let in_rate = source_packet.spec().rate; + + if in_rate == SAMPLE_RATE_RAW as u32 { + // No need to resample: mix as standard. + let samples_marched = mix_over_ref( + &source_packet, + symph_mix, + local_state.inner_pos, + samples_written, + volume, + ); + + samples_written += samples_marched; + + local_state.inner_pos += samples_marched; + local_state.inner_pos %= source_packet.frames(); + } else { + // NOTE: this should NEVER change in one stream. + let chan_c = source_packet.spec().channels.count(); + let (_, resampler, rs_out_buf) = local_state.resampler.get_or_insert_with(|| { + // TODO: integ. error handling here. + let resampler = FftFixedOut::new( + in_rate as usize, + SAMPLE_RATE_RAW, + RESAMPLE_OUTPUT_FRAME_SIZE, + 4, + chan_c, + ) + .expect("Failed to create resampler."); + let out_buf = resampler.output_buffer_allocate(); + + (chan_c, resampler, out_buf) + }); + + let inner_pos = local_state.inner_pos; + let pkt_frames = source_packet.frames(); + + if pkt_frames == 0 { + continue; + } + + let needed_in_frames = resampler.input_frames_next(); + let available_frames = pkt_frames - inner_pos; + + let force_copy = resample_in_progress || needed_in_frames > available_frames; + if (!force_copy) && matches!(source_packet, AudioBufferRef::F32(_)) { + // This is the only case where we can pull off a straight resample... + // I would really like if this could be a slice of slices, + // but the technology just isn't there yet. And I don't feel like + // writing unsafe transformations to do so. + + // NOTE: if let needed as if-let && {bool} is nightly only. + if let AudioBufferRef::F32(s_pkt) = source_packet { + let refs: Vec<&[f32]> = s_pkt + .planes() + .planes() + .iter() + .map(|s| &s[inner_pos..][..needed_in_frames]) + .collect(); + + local_state.inner_pos += needed_in_frames; + local_state.inner_pos %= pkt_frames; + + resampler + .process_into_buffer(&refs, rs_out_buf, None) + .unwrap(); + } else { + unreachable!() + } + } else { + // We either lack enough samples, or have the wrong data format, forcing + // a conversion/copy into the buffer. + let old_scratch_len = resample_scratch.frames(); + let missing_frames = needed_in_frames - old_scratch_len; + let frames_to_take = available_frames.min(missing_frames); + + resample_scratch.render_reserved(Some(frames_to_take)); + copy_into_resampler( + &source_packet, + resample_scratch, + inner_pos, + old_scratch_len, + frames_to_take, + ); + + local_state.inner_pos += frames_to_take; + local_state.inner_pos %= pkt_frames; + + if resample_scratch.frames() == needed_in_frames { + resampler + .process_into_buffer( + &resample_scratch.planes().planes()[..chan_c], + rs_out_buf, + None, + ) + .unwrap(); + resample_scratch.clear(); + resample_in_progress = false; + } else { + // Not enough data to fill the resampler: fetch more. + resample_in_progress = true; + continue; + } + }; + + let samples_marched = mix_resampled(rs_out_buf, symph_mix, samples_written, volume); + + samples_written += samples_marched; + } + } + + (MixType::MixedPcm(samples_written), track_status) +} + +#[inline] +fn mix_over_ref( + source: &AudioBufferRef, + target: &mut AudioBuffer, + source_pos: usize, + dest_pos: usize, + volume: f32, +) -> usize { + match source { + AudioBufferRef::U8(v) => mix_symph_buffer(v, target, source_pos, dest_pos, volume), + AudioBufferRef::U16(v) => mix_symph_buffer(v, target, source_pos, dest_pos, volume), + AudioBufferRef::U24(v) => mix_symph_buffer(v, target, source_pos, dest_pos, volume), + AudioBufferRef::U32(v) => mix_symph_buffer(v, target, source_pos, dest_pos, volume), + AudioBufferRef::S8(v) => mix_symph_buffer(v, target, source_pos, dest_pos, volume), + AudioBufferRef::S16(v) => mix_symph_buffer(v, target, source_pos, dest_pos, volume), + AudioBufferRef::S24(v) => mix_symph_buffer(v, target, source_pos, dest_pos, volume), + AudioBufferRef::S32(v) => mix_symph_buffer(v, target, source_pos, dest_pos, volume), + AudioBufferRef::F32(v) => mix_symph_buffer(v, target, source_pos, dest_pos, volume), + AudioBufferRef::F64(v) => mix_symph_buffer(v, target, source_pos, dest_pos, volume), + } +} + +#[inline] +fn mix_symph_buffer( + source: &AudioBuffer, + target: &mut AudioBuffer, + source_pos: usize, + dest_pos: usize, + volume: f32, +) -> usize +where + S: Sample + IntoSample, +{ + // mix in source_packet[inner_pos..] til end of EITHER buffer. + let src_usable = source.frames() - source_pos; + let tgt_usable = target.frames() - dest_pos; + + let mix_ct = src_usable.min(tgt_usable); + + let target_chans = target.spec().channels.count(); + let target_mono = target_chans == 1; + let source_chans = source.spec().channels.count(); + let source_mono = source_chans == 1; + + let source_planes = source.planes(); + let source_raw_planes = source_planes.planes(); + + if source_mono { + // mix this signal into *all* output channels at req'd volume. + let source_plane = source_raw_planes[0]; + for d_plane in (&mut *target.planes_mut().planes()).iter_mut() { + for (d, s) in d_plane[dest_pos..dest_pos + mix_ct] + .iter_mut() + .zip(source_plane[source_pos..source_pos + mix_ct].iter()) + { + *d += volume * (*s).into_sample(); + } + } + } else if target_mono { + // mix all signals into the one target channel: reduce aggregate volume + // by n_channels. + let vol_adj = 1.0 / (source_chans as f32); + let mut t_planes = target.planes_mut(); + let d_plane = &mut *t_planes.planes()[0]; + for s_plane in source_raw_planes[..].iter() { + for (d, s) in d_plane[dest_pos..dest_pos + mix_ct] + .iter_mut() + .zip(s_plane[source_pos..source_pos + mix_ct].iter()) + { + *d += volume * vol_adj * (*s).into_sample(); + } + } + } else { + // stereo -> stereo: don't change volume, map input -> output channels w/ no duplication + for (d_plane, s_plane) in (&mut *target.planes_mut().planes()) + .iter_mut() + .zip(source_raw_planes[..].iter()) + { + for (d, s) in d_plane[dest_pos..dest_pos + mix_ct] + .iter_mut() + .zip(s_plane[source_pos..source_pos + mix_ct].iter()) + { + *d += volume * (*s).into_sample(); + } + } + } + + mix_ct +} + +#[inline] +fn mix_resampled( + source: &[Vec], + target: &mut AudioBuffer, + dest_pos: usize, + volume: f32, +) -> usize { + let mix_ct = source[0].len(); + + let target_chans = target.spec().channels.count(); + let target_mono = target_chans == 1; + let source_chans = source.len(); + let source_mono = source_chans == 1; + + // see `mix_symph_buffer` for explanations of stereo<->mono logic. + if source_mono { + let source_plane = &source[0]; + for d_plane in (&mut *target.planes_mut().planes()).iter_mut() { + for (d, s) in d_plane[dest_pos..dest_pos + mix_ct] + .iter_mut() + .zip(source_plane) + { + *d += volume * s; + } + } + } else if target_mono { + let vol_adj = 1.0 / (source_chans as f32); + let mut t_planes = target.planes_mut(); + let d_plane = &mut *t_planes.planes()[0]; + for s_plane in source[..].iter() { + for (d, s) in d_plane[dest_pos..dest_pos + mix_ct].iter_mut().zip(s_plane) { + *d += volume * vol_adj * s; + } + } + } else { + for (d_plane, s_plane) in (&mut *target.planes_mut().planes()) + .iter_mut() + .zip(source[..].iter()) + { + for (d, s) in d_plane[dest_pos..dest_pos + mix_ct].iter_mut().zip(s_plane) { + *d += volume * (*s); + } + } + } + + mix_ct +} + +#[inline] +pub(crate) fn copy_into_resampler( + source: &AudioBufferRef, + target: &mut AudioBuffer, + source_pos: usize, + dest_pos: usize, + len: usize, +) -> usize { + match source { + AudioBufferRef::U8(v) => copy_symph_buffer(v, target, source_pos, dest_pos, len), + AudioBufferRef::U16(v) => copy_symph_buffer(v, target, source_pos, dest_pos, len), + AudioBufferRef::U24(v) => copy_symph_buffer(v, target, source_pos, dest_pos, len), + AudioBufferRef::U32(v) => copy_symph_buffer(v, target, source_pos, dest_pos, len), + AudioBufferRef::S8(v) => copy_symph_buffer(v, target, source_pos, dest_pos, len), + AudioBufferRef::S16(v) => copy_symph_buffer(v, target, source_pos, dest_pos, len), + AudioBufferRef::S24(v) => copy_symph_buffer(v, target, source_pos, dest_pos, len), + AudioBufferRef::S32(v) => copy_symph_buffer(v, target, source_pos, dest_pos, len), + AudioBufferRef::F32(v) => copy_symph_buffer(v, target, source_pos, dest_pos, len), + AudioBufferRef::F64(v) => copy_symph_buffer(v, target, source_pos, dest_pos, len), + } +} + +#[inline] +fn copy_symph_buffer( + source: &AudioBuffer, + target: &mut AudioBuffer, + source_pos: usize, + dest_pos: usize, + len: usize, +) -> usize +where + S: Sample + IntoSample, +{ + for (d_plane, s_plane) in (&mut *target.planes_mut().planes()) + .iter_mut() + .zip(source.planes().planes()[..].iter()) + { + for (d, s) in d_plane[dest_pos..dest_pos + len] + .iter_mut() + .zip(s_plane[source_pos..source_pos + len].iter()) + { + *d = (*s).into_sample(); + } + } + + len +} diff --git a/src/driver/tasks/mixer/mod.rs b/src/driver/tasks/mixer/mod.rs new file mode 100644 index 0000000..56c1c1d --- /dev/null +++ b/src/driver/tasks/mixer/mod.rs @@ -0,0 +1,924 @@ +pub mod mix_logic; +mod pool; +mod result; +pub mod state; +pub mod track; +mod util; + +use pool::*; +use result::*; +use state::*; +pub use track::*; + +use super::{disposal, error::Result, message::*}; +use crate::{ + constants::*, + driver::MixMode, + events::EventStore, + input::{Input, Parsed}, + tracks::{Action, LoopState, PlayError, PlayMode, TrackCommand, TrackHandle, TrackState, View}, + Config, +}; +use audiopus::{ + coder::Encoder as OpusEncoder, + softclip::SoftClip, + Application as CodingMode, + Bitrate, +}; +use discortp::{ + rtp::{MutableRtpPacket, RtpPacket}, + MutablePacket, +}; +use flume::{Receiver, Sender, TryRecvError}; +use rand::random; +use rubato::{FftFixedOut, Resampler}; +use std::{ + io::Write, + sync::Arc, + time::{Duration, Instant}, +}; +use symphonia_core::{ + audio::{AudioBuffer, AudioBufferRef, Layout, SampleBuffer, Signal, SignalSpec}, + codecs::CODEC_TYPE_OPUS, + conv::IntoSample, + formats::SeekTo, + sample::Sample, + units::Time, +}; +use tokio::runtime::Handle; +use tracing::{debug, error, instrument, warn}; +use xsalsa20poly1305::TAG_SIZE; + +#[cfg(test)] +use crate::driver::test_config::{OutputMessage, OutputMode, TickStyle}; +#[cfg(test)] +use discortp::Packet as _; + +pub struct Mixer { + pub bitrate: Bitrate, + pub config: Arc, + pub conn_active: Option, + pub content_prep_sequence: u64, + pub deadline: Instant, + pub disposer: Sender, + pub encoder: OpusEncoder, + pub interconnect: Interconnect, + pub mix_rx: Receiver, + pub muted: bool, + pub packet: [u8; VOICE_PACKET_MAX], + pub prevent_events: bool, + pub silence_frames: u8, + pub skip_sleep: bool, + pub soft_clip: SoftClip, + thread_pool: BlockyTaskPool, + pub ws: Option>, + + pub tracks: Vec, + track_handles: Vec, + + sample_buffer: SampleBuffer, + symph_mix: AudioBuffer, + resample_scratch: AudioBuffer, + + #[cfg(test)] + remaining_loops: Option, +} + +fn new_encoder(bitrate: Bitrate, mix_mode: MixMode) -> Result { + let mut encoder = OpusEncoder::new(SAMPLE_RATE, mix_mode.to_opus(), CodingMode::Audio)?; + encoder.set_bitrate(bitrate)?; + + Ok(encoder) +} + +impl Mixer { + pub fn new( + mix_rx: Receiver, + async_handle: Handle, + interconnect: Interconnect, + config: Config, + ) -> Self { + let bitrate = DEFAULT_BITRATE; + let encoder = new_encoder(bitrate, config.mix_mode) + .expect("Failed to create encoder in mixing thread with known-good values."); + let soft_clip = SoftClip::new(config.mix_mode.to_opus()); + + let mut packet = [0u8; VOICE_PACKET_MAX]; + + let mut rtp = MutableRtpPacket::new(&mut packet[..]).expect( + "FATAL: Too few bytes in self.packet for RTP header.\ + (Blame: VOICE_PACKET_MAX?)", + ); + rtp.set_version(RTP_VERSION); + rtp.set_payload_type(RTP_PROFILE_TYPE); + rtp.set_sequence(random::().into()); + rtp.set_timestamp(random::().into()); + + let tracks = Vec::with_capacity(1.max(config.preallocated_tracks)); + let track_handles = Vec::with_capacity(1.max(config.preallocated_tracks)); + + // Create an object disposal thread here. + let (disposer, disposal_rx) = flume::unbounded(); + std::thread::spawn(move || disposal::runner(disposal_rx)); + + let thread_pool = BlockyTaskPool::new(async_handle); + + let symph_layout = config.mix_mode.symph_layout(); + + let config = config.into(); + + let sample_buffer = SampleBuffer::::new( + MONO_FRAME_SIZE as u64, + symphonia_core::audio::SignalSpec::new_with_layout( + SAMPLE_RATE_RAW as u32, + symph_layout, + ), + ); + let symph_mix = AudioBuffer::::new( + MONO_FRAME_SIZE as u64, + symphonia_core::audio::SignalSpec::new_with_layout( + SAMPLE_RATE_RAW as u32, + symph_layout, + ), + ); + let resample_scratch = AudioBuffer::::new( + MONO_FRAME_SIZE as u64, + SignalSpec::new_with_layout(SAMPLE_RATE_RAW as u32, Layout::Stereo), + ); + + Self { + bitrate, + config, + conn_active: None, + content_prep_sequence: 0, + deadline: Instant::now(), + disposer, + encoder, + interconnect, + mix_rx, + muted: false, + packet, + prevent_events: false, + silence_frames: 0, + skip_sleep: false, + soft_clip, + thread_pool, + ws: None, + + tracks, + track_handles, + + sample_buffer, + symph_mix, + resample_scratch, + + #[cfg(test)] + remaining_loops: None, + } + } + + fn run(&mut self) { + let mut events_failure = false; + let mut conn_failure = false; + + #[cfg(test)] + let ignore_check = self.config.override_connection.is_some(); + #[cfg(not(test))] + let ignore_check = false; + + 'runner: loop { + if self.conn_active.is_some() || ignore_check { + loop { + match self.mix_rx.try_recv() { + Ok(m) => { + let (events, conn, should_exit) = self.handle_message(m); + events_failure |= events; + conn_failure |= conn; + + if should_exit { + break 'runner; + } + }, + + Err(TryRecvError::Disconnected) => { + break 'runner; + }, + + Err(TryRecvError::Empty) => { + break; + }, + }; + } + + // The above action may have invalidated the connection; need to re-check! + // Also, if we're in a test mode we should unconditionally run packet mixing code. + if self.conn_active.is_some() || ignore_check { + if let Err(e) = self.cycle().and_then(|_| self.audio_commands_events()) { + events_failure |= e.should_trigger_interconnect_rebuild(); + conn_failure |= e.should_trigger_connect(); + + debug!("Mixer thread cycle: {:?}", e); + } + } + } else { + match self.mix_rx.recv() { + Ok(m) => { + let (events, conn, should_exit) = self.handle_message(m); + events_failure |= events; + conn_failure |= conn; + + if should_exit { + break 'runner; + } + }, + Err(_) => { + break 'runner; + }, + } + } + + // event failure? rebuild interconnect. + // ws or udp failure? full connect + // (soft reconnect is covered by the ws task.) + // + // in both cases, send failure is fatal, + // but will only occur on disconnect. + // expecting this is fairly noisy, so exit silently. + if events_failure { + self.prevent_events = true; + let sent = self + .interconnect + .core + .send(CoreMessage::RebuildInterconnect); + events_failure = false; + + if sent.is_err() { + break; + } + } + + if conn_failure { + self.conn_active = None; + let sent = self.interconnect.core.send(CoreMessage::FullReconnect); + conn_failure = false; + + if sent.is_err() { + break; + } + } + } + } + + #[inline] + fn handle_message(&mut self, msg: MixerMessage) -> (bool, bool, bool) { + let mut events_failure = false; + let mut conn_failure = false; + let mut should_exit = false; + + let error = match msg { + MixerMessage::AddTrack(t) => self.add_track(t), + MixerMessage::SetTrack(t) => { + self.tracks.clear(); + + let mut out = self.fire_event(EventMessage::RemoveAllTracks); + + if let Some(t) = t { + // Do this unconditionally: this affects local state infallibly, + // with the event installation being the remote part. + if let Err(e) = self.add_track(t) { + out = Err(e); + } + } + + out + }, + MixerMessage::SetBitrate(b) => { + self.bitrate = b; + if let Err(e) = self.set_bitrate(b) { + error!("Failed to update bitrate {:?}", e); + } + Ok(()) + }, + MixerMessage::SetMute(m) => { + self.muted = m; + Ok(()) + }, + MixerMessage::SetConn(conn, ssrc) => { + self.conn_active = Some(conn); + let mut rtp = MutableRtpPacket::new(&mut self.packet[..]).expect( + "Too few bytes in self.packet for RTP header.\ + (Blame: VOICE_PACKET_MAX?)", + ); + rtp.set_ssrc(ssrc); + rtp.set_sequence(random::().into()); + rtp.set_timestamp(random::().into()); + self.deadline = Instant::now(); + Ok(()) + }, + MixerMessage::DropConn => { + self.conn_active = None; + Ok(()) + }, + MixerMessage::ReplaceInterconnect(i) => { + self.prevent_events = false; + if let Some(ws) = &self.ws { + conn_failure |= ws.send(WsMessage::ReplaceInterconnect(i.clone())).is_err(); + } + if let Some(conn) = &self.conn_active { + conn_failure |= conn + .udp_rx + .send(UdpRxMessage::ReplaceInterconnect(i.clone())) + .is_err(); + } + + self.interconnect = i; + + self.rebuild_tracks() + }, + MixerMessage::SetConfig(new_config) => { + if new_config.mix_mode != self.config.mix_mode { + self.soft_clip = SoftClip::new(new_config.mix_mode.to_opus()); + if let Ok(enc) = new_encoder(self.bitrate, new_config.mix_mode) { + self.encoder = enc; + } else { + self.bitrate = DEFAULT_BITRATE; + self.encoder = new_encoder(self.bitrate, new_config.mix_mode) + .expect("Failed fallback rebuild of OpusEncoder with safe inputs."); + } + + let sl = new_config.mix_mode.symph_layout(); + self.sample_buffer = SampleBuffer::::new( + MONO_FRAME_SIZE as u64, + SignalSpec::new_with_layout(SAMPLE_RATE_RAW as u32, sl), + ); + self.symph_mix = AudioBuffer::::new( + MONO_FRAME_SIZE as u64, + SignalSpec::new_with_layout(SAMPLE_RATE_RAW as u32, sl), + ); + } + + self.config = Arc::new(new_config.clone()); + + if self.tracks.capacity() < self.config.preallocated_tracks { + self.tracks + .reserve(self.config.preallocated_tracks - self.tracks.len()); + } + + if let Some(conn) = &self.conn_active { + conn_failure |= conn + .udp_rx + .send(UdpRxMessage::SetConfig(new_config)) + .is_err(); + } + + Ok(()) + }, + MixerMessage::RebuildEncoder => match new_encoder(self.bitrate, self.config.mix_mode) { + Ok(encoder) => { + self.encoder = encoder; + Ok(()) + }, + Err(e) => { + error!("Failed to rebuild encoder. Resetting bitrate. {:?}", e); + self.bitrate = DEFAULT_BITRATE; + self.encoder = new_encoder(self.bitrate, self.config.mix_mode) + .expect("Failed fallback rebuild of OpusEncoder with safe inputs."); + Ok(()) + }, + }, + MixerMessage::Ws(new_ws_handle) => { + self.ws = new_ws_handle; + Ok(()) + }, + MixerMessage::Poison => { + should_exit = true; + Ok(()) + }, + }; + + if let Err(e) = error { + events_failure |= e.should_trigger_interconnect_rebuild(); + conn_failure |= e.should_trigger_connect(); + } + + (events_failure, conn_failure, should_exit) + } + + #[inline] + fn fire_event(&self, event: EventMessage) -> Result<()> { + // As this task is responsible for noticing the potential death of an event context, + // it's responsible for not forcibly recreating said context repeatedly. + if !self.prevent_events { + self.interconnect.events.send(event)?; + } + + Ok(()) + } + + #[inline] + pub fn add_track(&mut self, track: TrackContext) -> Result<()> { + let (track, evts, state, handle) = InternalTrack::decompose_track(track); + self.tracks.push(track); + self.track_handles.push(handle.clone()); + self.interconnect + .events + .send(EventMessage::AddTrack(evts, state, handle))?; + + Ok(()) + } + + // rebuilds the event thread's view of each track, in event of a full rebuild. + #[inline] + fn rebuild_tracks(&mut self) -> Result<()> { + for (track, handle) in self.tracks.iter().zip(self.track_handles.iter()) { + let evts = EventStore::default(); + let state = track.state(); + let handle = handle.clone(); + + self.interconnect + .events + .send(EventMessage::AddTrack(evts, state, handle))?; + } + + Ok(()) + } + + #[inline] + fn audio_commands_events(&mut self) -> Result<()> { + // Apply user commands. + for (i, track) in self.tracks.iter_mut().enumerate() { + // This causes fallible event system changes, + // but if the event thread has died then we'll certainly + // detect that on the tick later. + // Changes to play state etc. MUST all be handled. + let action = track.process_commands(i, &self.interconnect); + + if let Some(req) = action.seek_point { + track.seek( + i, + req, + &self.interconnect, + &self.thread_pool, + &self.config, + self.prevent_events, + ); + } + + if let Some(callback) = action.make_playable { + if let Err(e) = track.get_or_ready_input( + i, + &self.interconnect, + &self.thread_pool, + &self.config, + self.prevent_events, + ) { + track.callbacks.make_playable = Some(callback); + if let Some(fail) = e.as_user() { + track.playing = PlayMode::Errored(fail); + } + if let Some(req) = e.into_seek_request() { + track.seek( + i, + req, + &self.interconnect, + &self.thread_pool, + &self.config, + self.prevent_events, + ); + } + } else { + // Track is already ready: don't register callback and just act. + drop(callback.send(Ok(()))); + } + } + } + + let mut i = 0; + while i < self.tracks.len() { + let track = self + .tracks + .get_mut(i) + .expect("Tried to remove an illegal track index."); + + if track.playing.is_done() { + let p_state = track.playing.clone(); + let to_drop = self.tracks.swap_remove(i); + drop( + self.disposer + .send(DisposalMessage::Track(Box::new(to_drop))), + ); + let to_drop = self.track_handles.swap_remove(i); + drop(self.disposer.send(DisposalMessage::Handle(to_drop))); + + self.fire_event(EventMessage::ChangeState( + i, + TrackStateChange::Mode(p_state), + ))?; + } else { + i += 1; + } + } + + // Tick -- receive side also handles removals in same manner after it increments + // times etc. + self.fire_event(EventMessage::Tick)?; + + Ok(()) + } + + #[cfg(test)] + fn _march_deadline(&mut self) { + match &self.config.tick_style { + TickStyle::Timed => { + std::thread::sleep(self.deadline.saturating_duration_since(Instant::now())); + self.deadline += TIMESTEP_LENGTH; + }, + TickStyle::UntimedWithExecLimit(rx) => { + if self.remaining_loops.is_none() { + if let Ok(new_val) = rx.recv() { + self.remaining_loops = Some(new_val.wrapping_sub(1)); + } + } + + if let Some(cnt) = self.remaining_loops.as_mut() { + if *cnt == 0 { + self.remaining_loops = None; + } else { + *cnt = cnt.wrapping_sub(1); + } + } + }, + } + } + + #[cfg(not(test))] + #[inline(always)] + #[allow(clippy::inline_always)] // Justified, this is a very very hot path + fn _march_deadline(&mut self) { + std::thread::sleep(self.deadline.saturating_duration_since(Instant::now())); + self.deadline += TIMESTEP_LENGTH; + } + + #[inline] + fn march_deadline(&mut self) { + if self.skip_sleep { + return; + } + + self._march_deadline(); + } + + pub fn cycle(&mut self) -> Result<()> { + let mut mix_buffer = [0f32; STEREO_FRAME_SIZE]; + + // symph_mix is an `AudioBuffer` (planar format), we need to convert this + // later into an interleaved `SampleBuffer` for libopus. + self.symph_mix.clear(); + self.symph_mix.render_reserved(Some(MONO_FRAME_SIZE)); + self.resample_scratch.clear(); + + // Walk over all the audio files, combining into one audio frame according + // to volume, play state, etc. + let mut mix_len = { + let out = self.mix_tracks(); + + self.sample_buffer.copy_interleaved_typed(&self.symph_mix); + + out + }; + + if self.muted { + mix_len = MixType::MixedPcm(0); + } + + // Explicit "Silence" frame handling: if there is no mixed data, we must send + // ~5 frames of silence (unless another good audio frame appears) before we + // stop sending RTP frames. + if mix_len == MixType::MixedPcm(0) { + if self.silence_frames > 0 { + self.silence_frames -= 1; + let mut rtp = MutableRtpPacket::new(&mut self.packet[..]).expect( + "FATAL: Too few bytes in self.packet for RTP header.\ + (Blame: VOICE_PACKET_MAX?)", + ); + + let payload = rtp.payload_mut(); + + (&mut payload[TAG_SIZE..TAG_SIZE + SILENT_FRAME.len()]) + .copy_from_slice(&SILENT_FRAME[..]); + + mix_len = MixType::Passthrough(SILENT_FRAME.len()); + } else { + // Per official guidelines, send 5x silence BEFORE we stop speaking. + if let Some(ws) = &self.ws { + // NOTE: this explicit `drop` should prevent a catastrophic thread pileup. + // A full reconnect might cause an inner closed connection. + // It's safer to leave the central task to clean this up and + // pass the mixer a new channel. + drop(ws.send(WsMessage::Speaking(false))); + } + + self.march_deadline(); + + #[cfg(test)] + match &self.config.override_connection { + Some(OutputMode::Raw(tx)) => + drop(tx.send(crate::driver::test_config::TickMessage::NoEl)), + Some(OutputMode::Rtp(tx)) => + drop(tx.send(crate::driver::test_config::TickMessage::NoEl)), + None => {}, + } + + return Ok(()); + } + } else { + self.silence_frames = 5; + + if let MixType::MixedPcm(n) = mix_len { + // FIXME: When impling #134, prevent this copy from happening if softclip disabled. + // Offer sample_buffer.samples() to prep_and_send_packet. + + // to apply soft_clip, we need this to be in a normal f32 buffer. + // unfortunately, SampleBuffer does not expose a `.samples_mut()`. + // hence, an extra copy... + let samples_to_copy = self.config.mix_mode.channels() * n; + + (&mut mix_buffer[..samples_to_copy]) + .copy_from_slice(&self.sample_buffer.samples()[..samples_to_copy]); + + self.soft_clip.apply( + (&mut mix_buffer[..]) + .try_into() + .expect("Mix buffer is known to have a valid sample count (softclip)."), + )?; + } + } + + if let Some(ws) = &self.ws { + ws.send(WsMessage::Speaking(true))?; + } + + // Wait till the right time to send this packet: + // usually a 20ms tick, in test modes this is either a finite number of runs or user input. + self.march_deadline(); + + #[cfg(test)] + if let Some(OutputMode::Raw(tx)) = &self.config.override_connection { + let msg = match mix_len { + MixType::Passthrough(len) if len == SILENT_FRAME.len() => OutputMessage::Silent, + MixType::Passthrough(len) => { + let rtp = RtpPacket::new(&self.packet[..]).expect( + "FATAL: Too few bytes in self.packet for RTP header.\ + (Blame: VOICE_PACKET_MAX?)", + ); + let payload = rtp.payload(); + let opus_frame = (&payload[TAG_SIZE..][..len]).to_vec(); + + OutputMessage::Passthrough(opus_frame) + }, + MixType::MixedPcm(_) => OutputMessage::Mixed( + mix_buffer[..self.config.mix_mode.sample_count_in_frame()].to_vec(), + ), + }; + + drop(tx.send(msg.into())); + } else { + self.prep_and_send_packet(&mix_buffer, mix_len)?; + } + + #[cfg(not(test))] + self.prep_and_send_packet(&mix_buffer, mix_len)?; + + // Zero out all planes of the mix buffer if any audio was written. + if matches!(mix_len, MixType::MixedPcm(a) if a > 0) { + for plane in self.symph_mix.planes_mut().planes() { + plane.fill(0.0); + } + } + + Ok(()) + } + + fn set_bitrate(&mut self, bitrate: Bitrate) -> Result<()> { + self.encoder.set_bitrate(bitrate).map_err(Into::into) + } + + #[inline] + fn prep_and_send_packet(&mut self, buffer: &[f32; 1920], mix_len: MixType) -> Result<()> { + let conn = self + .conn_active + .as_mut() + .expect("Shouldn't be mixing packets without access to a cipher + UDP dest."); + + let index = { + let mut rtp = MutableRtpPacket::new(&mut self.packet[..]).expect( + "FATAL: Too few bytes in self.packet for RTP header.\ + (Blame: VOICE_PACKET_MAX?)", + ); + + let payload = rtp.payload_mut(); + let crypto_mode = conn.crypto_state.kind(); + + // If passthrough, Opus payload in place already. + // Else encode into buffer with space for AEAD encryption headers. + let payload_len = match mix_len { + MixType::Passthrough(opus_len) => opus_len, + MixType::MixedPcm(_samples) => { + let total_payload_space = payload.len() - crypto_mode.payload_suffix_len(); + self.encoder.encode_float( + &buffer[..self.config.mix_mode.sample_count_in_frame()], + &mut payload[TAG_SIZE..total_payload_space], + )? + }, + }; + + let final_payload_size = conn + .crypto_state + .write_packet_nonce(&mut rtp, TAG_SIZE + payload_len); + + // Packet encryption ignored in test modes. + #[cfg(not(test))] + let encrypt = true; + #[cfg(test)] + let encrypt = self.config.override_connection.is_none(); + + if encrypt { + conn.crypto_state.kind().encrypt_in_place( + &mut rtp, + &conn.cipher, + final_payload_size, + )?; + } + + RtpPacket::minimum_packet_size() + final_payload_size + }; + + #[cfg(test)] + if let Some(OutputMode::Rtp(tx)) = &self.config.override_connection { + // Test mode: send unencrypted (compressed) packets to local receiver. + drop(tx.send(self.packet[..index].to_vec().into())); + } else { + conn.udp_tx.send(self.packet[..index].to_vec())?; + } + + #[cfg(not(test))] + { + // Normal operation: send encrypted payload to UDP Tx task. + + // TODO: This is dog slow, don't do this. + // Can we replace this with a shared ring buffer + semaphore? + // or the BBQueue crate? + conn.udp_tx.send(self.packet[..index].to_vec())?; + } + + let mut rtp = MutableRtpPacket::new(&mut self.packet[..]).expect( + "FATAL: Too few bytes in self.packet for RTP header.\ + (Blame: VOICE_PACKET_MAX?)", + ); + rtp.set_sequence(rtp.get_sequence() + 1); + rtp.set_timestamp(rtp.get_timestamp() + MONO_FRAME_SIZE as u32); + + Ok(()) + } + + #[inline] + fn mix_tracks(&mut self) -> MixType { + // Get a slice of bytes to write in data for Opus packet passthrough. + let mut rtp = MutableRtpPacket::new(&mut self.packet[..]).expect( + "FATAL: Too few bytes in self.packet for RTP header.\ + (Blame: VOICE_PACKET_MAX?)", + ); + let payload = rtp.payload_mut(); + let opus_frame = &mut payload[TAG_SIZE..]; + + // Opus frame passthrough. + // This requires that we have only one PLAYING track, who has volume 1.0, and an + // Opus codec type (verified later in mix_symph_indiv). + // + // We *could* cache the number of live tracks separately, but that makes this + // quite fragile given all the ways a user can alter the PlayMode. + let mut num_live = 0; + let mut last_live_vol = 1.0; + for track in &self.tracks { + if track.playing.is_playing() { + num_live += 1; + last_live_vol = track.volume; + } + } + let do_passthrough = num_live == 1 && (last_live_vol - 1.0).abs() < f32::EPSILON; + + let mut len = 0; + for (i, track) in self.tracks.iter_mut().enumerate() { + let vol = track.volume; + + // This specifically tries to get tracks who are "preparing", + // so that event handlers and the like can all be fired without + // the track being in a `Play` state. + if !track.should_check_input() { + continue; + } + + let should_play = track.playing.is_playing(); + + let input = track.get_or_ready_input( + i, + &self.interconnect, + &self.thread_pool, + &self.config, + self.prevent_events, + ); + + let (input, mix_state) = match input { + Ok(i) => i, + Err(InputReadyingError::Waiting) => continue, + Err(InputReadyingError::NeedsSeek(req)) => { + track.seek( + i, + req, + &self.interconnect, + &self.thread_pool, + &self.config, + self.prevent_events, + ); + continue; + }, + // TODO: allow for retry in given time. + Err(e) => { + if let Some(fail) = e.as_user() { + track.playing = PlayMode::Errored(fail); + } + continue; + }, + }; + + // Now that we have dealt with potential errors in preparing tracks, + // only do any mixing if the track is to be played! + if !should_play { + continue; + } + + let (mix_type, status) = mix_logic::mix_symph_indiv( + &mut self.symph_mix, + &mut self.resample_scratch, + input, + mix_state, + vol, + do_passthrough.then(|| &mut *opus_frame), + ); + + let return_here = if let MixType::MixedPcm(pcm_len) = mix_type { + len = len.max(pcm_len); + false + } else { + if mix_state.passthrough == Passthrough::Inactive { + input.decoder.reset(); + } + mix_state.passthrough = Passthrough::Active; + true + }; + + // FIXME: allow Ended to trigger a seek/loop/revisit in the same mix cycle? + // Would this be possible with special-casing to mark some inputs as fast + // to recreate? Probably not doable in the general case. + match status { + MixStatus::Live => track.step_frame(), + MixStatus::Errored(e) => + track.playing = PlayMode::Errored(PlayError::Decode(e.into())), + MixStatus::Ended if track.do_loop() => { + drop(self.track_handles[i].seek(Duration::default())); + if !self.prevent_events { + // position update is sent out later, when the seek concludes. + drop(self.interconnect.events.send(EventMessage::ChangeState( + i, + TrackStateChange::Loops(track.loops, false), + ))); + } + }, + MixStatus::Ended => { + track.end(); + }, + } + + // This needs to happen here due to borrow checker shenanigans. + if return_here { + return mix_type; + } + } + + MixType::MixedPcm(len) + } +} + +/// The mixing thread is a synchronous context due to its compute-bound nature. +/// +/// We pass in an async handle for the benefit of some Input classes (e.g., restartables) +/// who need to run their restart code elsewhere and return blank data until such time. +#[instrument(skip(interconnect, mix_rx, async_handle))] +pub(crate) fn runner( + interconnect: Interconnect, + mix_rx: Receiver, + async_handle: Handle, + config: Config, +) { + Mixer::new(mix_rx, async_handle, interconnect, config).run(); +} diff --git a/src/driver/tasks/mixer/pool.rs b/src/driver/tasks/mixer/pool.rs new file mode 100644 index 0000000..ff32a2e --- /dev/null +++ b/src/driver/tasks/mixer/pool.rs @@ -0,0 +1,148 @@ +use super::util::copy_seek_to; + +use crate::{ + driver::tasks::message::MixerInputResultMessage, + input::{AudioStream, AudioStreamError, Compose, Input, LiveInput, Parsed}, + Config, +}; +use flume::Sender; +use parking_lot::RwLock; +use std::{result::Result as StdResult, sync::Arc, time::Duration}; +use symphonia_core::{ + formats::{SeekMode, SeekTo}, + io::MediaSource, +}; +use tokio::runtime::Handle; + +#[derive(Clone)] +pub struct BlockyTaskPool { + pool: Arc>, + handle: Handle, +} + +impl BlockyTaskPool { + pub fn new(handle: Handle) -> Self { + Self { + pool: Arc::new(RwLock::new(rusty_pool::ThreadPool::new( + 1, + 64, + Duration::from_secs(300), + ))), + handle, + } + } + + pub fn create( + &self, + callback: Sender, + input: Input, + seek_time: Option, + config: Arc, + ) { + // Moves an Input from Lazy -> Live. + // We either do this on this pool, or move it to the tokio executor as the source requires. + // This takes a seek_time to pass on and execute *after* parsing (i.e., back-seek on + // read-only stream). + match input { + Input::Lazy(mut lazy) => { + let far_pool = self.clone(); + if lazy.should_create_async() { + self.handle.spawn(async move { + let out = lazy.create_async().await; + far_pool.send_to_parse(out, lazy, callback, seek_time, config); + }); + } else { + let pool = self.pool.read(); + pool.execute(move || { + let out = lazy.create(); + far_pool.send_to_parse(out, lazy, callback, seek_time, config); + }); + } + }, + Input::Live(live, maybe_create) => + self.parse(config, callback, live, maybe_create, seek_time), + } + } + + pub fn send_to_parse( + &self, + create_res: StdResult>, AudioStreamError>, + rec: Box, + callback: Sender, + seek_time: Option, + config: Arc, + ) { + match create_res { + Ok(o) => { + self.parse(config, callback, LiveInput::Raw(o), Some(rec), seek_time); + }, + Err(e) => { + drop(callback.send(MixerInputResultMessage::CreateErr(e.into()))); + }, + } + } + + pub fn parse( + &self, + config: Arc, + callback: Sender, + input: LiveInput, + rec: Option>, + seek_time: Option, + ) { + let pool_clone = self.clone(); + let pool = self.pool.read(); + + pool.execute( + move || match input.promote(config.codec_registry, config.format_registry) { + Ok(LiveInput::Parsed(parsed)) => match seek_time { + // If seek time is zero, then wipe it out. + // Some formats (MKV) make SeekTo(0) require a backseek to realign with the + // current page. + Some(seek_time) if !super::util::seek_to_is_zero(&seek_time) => { + pool_clone.seek(callback, parsed, rec, seek_time, false, config); + }, + _ => { + drop(callback.send(MixerInputResultMessage::Built(parsed, rec))); + }, + }, + Ok(_) => unreachable!(), + Err(e) => { + drop(callback.send(MixerInputResultMessage::ParseErr(e.into()))); + }, + }, + ); + } + + pub fn seek( + &self, + callback: Sender, + mut input: Parsed, + rec: Option>, + seek_time: SeekTo, + // Not all of symphonia's formats bother to return SeekErrorKind::ForwardOnly. + // So, we need *this* flag. + backseek_needed: bool, + config: Arc, + ) { + let pool_clone = self.clone(); + let pool = self.pool.read(); + + pool.execute(move || match rec { + Some(rec) if (!input.supports_backseek) && backseek_needed => { + pool_clone.create(callback, Input::Lazy(rec), Some(seek_time), config); + }, + _ => { + let seek_result = input + .format + .seek(SeekMode::Accurate, copy_seek_to(&seek_time)); + input.decoder.reset(); + drop(callback.send(MixerInputResultMessage::Seek( + input, + rec, + seek_result.map_err(Arc::new), + ))); + }, + }); + } +} diff --git a/src/driver/tasks/mixer/result.rs b/src/driver/tasks/mixer/result.rs new file mode 100644 index 0000000..1957d33 --- /dev/null +++ b/src/driver/tasks/mixer/result.rs @@ -0,0 +1,55 @@ +use crate::{ + input::AudioStreamError, + tracks::{PlayError, SeekRequest}, +}; +use std::sync::Arc; +use symphonia_core::errors::Error as SymphoniaError; + +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum MixType { + Passthrough(usize), + MixedPcm(usize), +} + +pub enum MixStatus { + Live, + Ended, + Errored(SymphoniaError), +} + +impl From for MixStatus { + fn from(e: SymphoniaError) -> Self { + Self::Errored(e) + } +} + +// The Symph errors are Arc'd here since if they come up, they will always +// be Arc'd anyway via into_user. +#[derive(Clone, Debug)] +pub enum InputReadyingError { + Parsing(Arc), + Creation(Arc), + Seeking(Arc), + Dropped, + Waiting, + NeedsSeek(SeekRequest), +} + +impl InputReadyingError { + pub fn as_user(&self) -> Option { + match self { + Self::Parsing(e) => Some(PlayError::Parse(e.clone())), + Self::Creation(e) => Some(PlayError::Create(e.clone())), + Self::Seeking(e) => Some(PlayError::Seek(e.clone())), + _ => None, + } + } + + pub fn into_seek_request(self) -> Option { + if let Self::NeedsSeek(a) = self { + Some(a) + } else { + None + } + } +} diff --git a/src/driver/tasks/mixer/state.rs b/src/driver/tasks/mixer/state.rs new file mode 100644 index 0000000..e5708f3 --- /dev/null +++ b/src/driver/tasks/mixer/state.rs @@ -0,0 +1,104 @@ +use crate::{ + constants::OPUS_PASSTHROUGH_STRIKE_LIMIT, + driver::tasks::message::*, + input::{Compose, Input, LiveInput, Metadata, Parsed}, + tracks::{ReadyState, SeekRequest}, +}; +use flume::Receiver; +use rubato::FftFixedOut; +use std::time::Instant; + +pub enum InputState { + NotReady(Input), + Preparing(PreparingInfo), + Ready(Parsed, Option>), +} + +impl InputState { + pub fn metadata(&mut self) -> Option { + if let Self::Ready(parsed, _) = self { + Some(parsed.into()) + } else { + None + } + } + + pub fn ready_state(&self) -> ReadyState { + match self { + Self::NotReady(_) => ReadyState::Uninitialised, + Self::Preparing(_) => ReadyState::Preparing, + Self::Ready(_, _) => ReadyState::Playable, + } + } +} + +impl From for InputState { + fn from(val: Input) -> Self { + match val { + a @ Input::Lazy(_) => Self::NotReady(a), + Input::Live(live, rec) => match live { + LiveInput::Parsed(p) => Self::Ready(p, rec), + other => Self::NotReady(Input::Live(other, rec)), + }, + } + } +} + +pub struct PreparingInfo { + #[allow(dead_code)] + /// Time this request was fired. + pub time: Instant, + /// Used to handle seek requests fired while a track was being created (or a seek was in progress). + pub queued_seek: Option, + /// Callback from the thread pool to indicate the result of creating/parsing this track. + pub callback: Receiver, +} + +pub struct DecodeState { + pub inner_pos: usize, + pub resampler: Option<(usize, FftFixedOut, Vec>)>, + pub passthrough: Passthrough, + pub passthrough_violations: u8, +} + +impl DecodeState { + pub fn reset(&mut self) { + self.inner_pos = 0; + self.resampler = None; + } + + pub fn record_and_check_passthrough_strike_final(&mut self, fatal: bool) -> bool { + self.passthrough_violations = self.passthrough_violations.saturating_add(1); + let blocked = fatal || self.passthrough_violations > OPUS_PASSTHROUGH_STRIKE_LIMIT; + if blocked { + self.passthrough = Passthrough::Block; + } + blocked + } +} + +impl Default for DecodeState { + fn default() -> Self { + Self { + inner_pos: 0, + resampler: None, + passthrough: Passthrough::Inactive, + passthrough_violations: 0, + } + } +} + +/// Simple state to manage decoder resets etc. +/// +/// Inactive->Active transitions should trigger a reset. +/// +/// Block should be used if a source contains known-bad packets: +/// it's unlikely that packet sizes will vary, but if they do then +/// we can't passthrough (and every attempt will trigger a codec reset, +/// which probably won't sound too smooth). +#[derive(Clone, Copy, Eq, PartialEq)] +pub enum Passthrough { + Active, + Inactive, + Block, +} diff --git a/src/driver/tasks/mixer/track.rs b/src/driver/tasks/mixer/track.rs new file mode 100644 index 0000000..259d66d --- /dev/null +++ b/src/driver/tasks/mixer/track.rs @@ -0,0 +1,400 @@ +use crate::tracks::{ReadyState, SeekRequest}; +use std::result::Result as StdResult; +use symphonia_core::errors::Error as SymphError; + +use super::*; + +pub struct InternalTrack { + pub(crate) playing: PlayMode, + pub(crate) volume: f32, + pub(crate) input: InputState, + pub(crate) mix_state: DecodeState, + pub(crate) position: Duration, + pub(crate) play_time: Duration, + pub(crate) commands: Receiver, + pub(crate) loops: LoopState, + pub(crate) callbacks: Callbacks, +} + +impl<'a> InternalTrack { + pub(crate) fn decompose_track( + val: TrackContext, + ) -> (Self, EventStore, TrackState, TrackHandle) { + let TrackContext { + handle, + track, + receiver, + } = val; + let out = InternalTrack { + playing: track.playing, + volume: track.volume, + input: InputState::from(track.input), + mix_state: DecodeState::default(), + position: Duration::default(), + play_time: Duration::default(), + commands: receiver, + loops: track.loops, + callbacks: Callbacks::default(), + }; + + let state = out.state(); + + (out, track.events, state, handle) + } + + pub(crate) fn state(&self) -> TrackState { + let ready = self.input.ready_state(); + + TrackState { + playing: self.playing.clone(), + volume: self.volume, + position: self.position, + play_time: self.play_time, + loops: self.loops, + ready, + } + } + + pub(crate) fn view(&'a mut self) -> View<'a> { + let ready = self.input.ready_state(); + + View { + position: &self.position, + play_time: &self.play_time, + volume: &mut self.volume, + meta: self.input.metadata(), + ready, + playing: &mut self.playing, + loops: &mut self.loops, + } + } + + pub(crate) fn process_commands(&mut self, index: usize, ic: &Interconnect) -> Action { + // Note: disconnection and an empty channel are both valid, + // and should allow the audio object to keep running as intended. + + // We also need to export a target seek point to the mixer, if known. + let mut action = Action::default(); + + // Note that interconnect failures are not currently errors. + // In correct operation, the event thread should never panic, + // but it receiving status updates is secondary do actually + // doing the work. + while let Ok(cmd) = self.commands.try_recv() { + match cmd { + TrackCommand::Play => { + self.playing.change_to(PlayMode::Play); + drop(ic.events.send(EventMessage::ChangeState( + index, + TrackStateChange::Mode(self.playing.clone()), + ))); + }, + TrackCommand::Pause => { + self.playing.change_to(PlayMode::Pause); + drop(ic.events.send(EventMessage::ChangeState( + index, + TrackStateChange::Mode(self.playing.clone()), + ))); + }, + TrackCommand::Stop => { + self.playing.change_to(PlayMode::Stop); + drop(ic.events.send(EventMessage::ChangeState( + index, + TrackStateChange::Mode(self.playing.clone()), + ))); + }, + TrackCommand::Volume(vol) => { + self.volume = vol; + drop(ic.events.send(EventMessage::ChangeState( + index, + TrackStateChange::Volume(self.volume), + ))); + }, + TrackCommand::Seek(req) => action.seek_point = Some(req), + TrackCommand::AddEvent(evt) => { + drop(ic.events.send(EventMessage::AddTrackEvent(index, evt))); + }, + TrackCommand::Do(func) => { + if let Some(indiv_action) = func(self.view()) { + action.combine(indiv_action); + } + + drop(ic.events.send(EventMessage::ChangeState( + index, + TrackStateChange::Total(self.state()), + ))); + }, + TrackCommand::Request(tx) => { + drop(tx.send(self.state())); + }, + TrackCommand::Loop(loops) => { + self.loops = loops; + drop(ic.events.send(EventMessage::ChangeState( + index, + TrackStateChange::Loops(self.loops, true), + ))); + }, + TrackCommand::MakePlayable(callback) => action.make_playable = Some(callback), + } + } + + action + } + + pub(crate) fn do_loop(&mut self) -> bool { + match self.loops { + LoopState::Infinite => true, + LoopState::Finite(0) => false, + LoopState::Finite(ref mut n) => { + *n -= 1; + true + }, + } + } + + /// Steps playback location forward by one frame. + pub(crate) fn step_frame(&mut self) { + self.position += TIMESTEP_LENGTH; + self.play_time += TIMESTEP_LENGTH; + } + + pub(crate) fn should_check_input(&self) -> bool { + self.playing.is_playing() || matches!(self.input, InputState::Preparing(_)) + } + + pub(crate) fn end(&mut self) -> &mut Self { + self.playing.change_to(PlayMode::End); + + self + } + + /// Readies the requested input state. + /// + /// Returns the usable version of the audio if available, and whether the track should be deleted. + pub(crate) fn get_or_ready_input( + &'a mut self, + id: usize, + interconnect: &Interconnect, + pool: &BlockyTaskPool, + config: &Arc, + prevent_events: bool, + ) -> StdResult<(&'a mut Parsed, &'a mut DecodeState), InputReadyingError> { + let input = &mut self.input; + let mix_state = &mut self.mix_state; + + let (out, queued_seek) = match input { + InputState::NotReady(_) => { + let (tx, rx) = flume::bounded(1); + + let mut state = InputState::Preparing(PreparingInfo { + time: Instant::now(), + queued_seek: None, + callback: rx, + }); + + std::mem::swap(&mut state, input); + + match state { + InputState::NotReady(a @ Input::Lazy(_)) => { + pool.create(tx, a, None, config.clone()); + }, + InputState::NotReady(Input::Live(audio, rec)) => { + pool.parse(config.clone(), tx, audio, rec, None); + }, + _ => unreachable!(), + } + + if !prevent_events { + drop(interconnect.events.send(EventMessage::ChangeState( + id, + TrackStateChange::Ready(ReadyState::Preparing), + ))); + } + + (Err(InputReadyingError::Waiting), None) + }, + InputState::Preparing(info) => { + let queued_seek = info.queued_seek.take(); + + let orig_out = match info.callback.try_recv() { + Ok(MixerInputResultMessage::Built(parsed, rec)) => { + *input = InputState::Ready(parsed, rec); + mix_state.reset(); + + // possible TODO: set position to the true track position here? + // ISSUE: need to get next_packet to see its `ts`, but inner_pos==0 + // will trigger next packet to be taken at mix time. + + if !prevent_events { + drop(interconnect.events.send(EventMessage::ChangeState( + id, + TrackStateChange::Ready(ReadyState::Playable), + ))); + } + + self.callbacks.playable(); + + if let InputState::Ready(ref mut parsed, _) = input { + Ok(parsed) + } else { + unreachable!() + } + }, + Ok(MixerInputResultMessage::Seek(parsed, rec, seek_res)) => { + match seek_res { + Ok(pos) => + if let Some(time_base) = parsed.decoder.codec_params().time_base { + // Update track's position to match the actual timestamp the + // seek landed at. + let new_time = time_base.calc_time(pos.actual_ts); + let time_in_float = new_time.seconds as f64 + new_time.frac; + self.position = + std::time::Duration::from_secs_f64(time_in_float); + + self.callbacks.seeked(self.position); + self.callbacks.playable(); + + if !prevent_events { + drop(interconnect.events.send(EventMessage::ChangeState( + id, + TrackStateChange::Position(self.position), + ))); + + drop(interconnect.events.send(EventMessage::ChangeState( + id, + TrackStateChange::Ready(ReadyState::Playable), + ))); + } + + // Our decoder state etc. must be reset. + // (Symphonia decoder state reset in the thread pool during + // the operation.) + mix_state.reset(); + *input = InputState::Ready(parsed, rec); + + if let InputState::Ready(ref mut parsed, _) = input { + Ok(parsed) + } else { + unreachable!() + } + } else { + Err(InputReadyingError::Seeking( + SymphError::Unsupported("Track had no recorded time base.") + .into(), + )) + }, + Err(e) => Err(InputReadyingError::Seeking(e)), + } + }, + Ok(MixerInputResultMessage::CreateErr(e)) => + Err(InputReadyingError::Creation(e)), + Ok(MixerInputResultMessage::ParseErr(e)) => Err(InputReadyingError::Parsing(e)), + Err(TryRecvError::Disconnected) => Err(InputReadyingError::Dropped), + Err(TryRecvError::Empty) => Err(InputReadyingError::Waiting), + }; + + let orig_out = orig_out.map(|a| (a, mix_state)); + + if let Err(ref e) = orig_out { + if let Some(e) = e.as_user() { + self.callbacks.readying_error(e); + } + } + + (orig_out, queued_seek) + }, + InputState::Ready(ref mut parsed, _) => (Ok((parsed, mix_state)), None), + }; + + match (out, queued_seek) { + (Ok(_), Some(request)) => Err(InputReadyingError::NeedsSeek(request)), + (a, _) => a, + } + } + + pub(crate) fn seek( + &mut self, + id: usize, + request: SeekRequest, + interconnect: &Interconnect, + pool: &BlockyTaskPool, + config: &Arc, + prevent_events: bool, + ) { + if let InputState::Preparing(p) = &mut self.input { + p.queued_seek = Some(request); + return; + } + + // might be a little topsy turvy: rethink me. + let SeekRequest { time, callback } = request; + + self.callbacks.seek = Some(callback); + if !prevent_events { + drop(interconnect.events.send(EventMessage::ChangeState( + id, + TrackStateChange::Ready(ReadyState::Preparing), + ))); + } + + let backseek_needed = time < self.position; + + let time = Time::from(time.as_secs_f64()); + let mut ts = SeekTo::Time { + time, + track_id: None, + }; + let (tx, rx) = flume::bounded(1); + + let state = std::mem::replace( + &mut self.input, + InputState::Preparing(PreparingInfo { + time: Instant::now(), + callback: rx, + queued_seek: None, + }), + ); + + match state { + InputState::Ready(p, r) => { + if let SeekTo::Time { time: _, track_id } = &mut ts { + *track_id = Some(p.track_id); + } + + pool.seek(tx, p, r, ts, backseek_needed, config.clone()); + }, + InputState::NotReady(lazy) => pool.create(tx, lazy, Some(ts), config.clone()), + InputState::Preparing(_) => unreachable!(), // Covered above. + } + } +} + +#[derive(Debug, Default)] +pub struct Callbacks { + pub seek: Option>>, + pub make_playable: Option>>, +} + +impl Callbacks { + fn readying_error(&mut self, err: PlayError) { + if let Some(callback) = self.seek.take() { + drop(callback.send(Err(err.clone()))); + } + + if let Some(callback) = self.make_playable.take() { + drop(callback.send(Err(err))); + } + } + + fn playable(&mut self) { + if let Some(callback) = self.make_playable.take() { + drop(callback.send(Ok(()))); + } + } + + fn seeked(&mut self, time: Duration) { + if let Some(callback) = self.seek.take() { + drop(callback.send(Ok(time))); + } + } +} diff --git a/src/driver/tasks/mixer/util.rs b/src/driver/tasks/mixer/util.rs new file mode 100644 index 0000000..3dcb059 --- /dev/null +++ b/src/driver/tasks/mixer/util.rs @@ -0,0 +1,20 @@ +use symphonia_core::{formats::SeekTo, units::Time}; + +// SeekTo lacks Copy and Clone... somehow. +pub fn copy_seek_to(pos: &SeekTo) -> SeekTo { + match *pos { + SeekTo::Time { time, track_id } => SeekTo::Time { time, track_id }, + SeekTo::TimeStamp { ts, track_id } => SeekTo::TimeStamp { ts, track_id }, + } +} + +pub fn seek_to_is_zero(pos: &SeekTo) -> bool { + match *pos { + SeekTo::Time { time, .. } => + time == Time { + seconds: 0, + frac: 0.0, + }, + SeekTo::TimeStamp { ts, .. } => ts == 0, + } +} diff --git a/src/driver/tasks/mod.rs b/src/driver/tasks/mod.rs index efe58ce..945c6e2 100644 --- a/src/driver/tasks/mod.rs +++ b/src/driver/tasks/mod.rs @@ -21,7 +21,7 @@ use crate::{ Config, ConnectionInfo, }; -use flume::{Receiver, RecvError, Sender}; +use flume::{Receiver, Sender}; use message::*; use tokio::{runtime::Handle, spawn, time::sleep as tsleep}; use tracing::{debug, instrument, trace}; @@ -70,23 +70,21 @@ async fn runner(mut config: Config, rx: Receiver, tx: Sender { + while let Ok(msg) = rx.recv_async().await { + match msg { + CoreMessage::ConnectWithResult(info, tx) => { config = if let Some(new_config) = next_config.take() { - let _ = interconnect - .mixer - .send(MixerMessage::SetConfig(new_config.clone())); + drop( + interconnect + .mixer + .send(MixerMessage::SetConfig(new_config.clone())), + ); new_config } else { config }; - if connection - .as_ref() - .map(|conn| conn.info != info) - .unwrap_or(true) - { + if connection.as_ref().map_or(true, |conn| conn.info != info) { // Only *actually* reconnect if the conn info changed, or we don't have an // active connection. // This allows the gateway component to keep sending join requests independent @@ -97,10 +95,10 @@ async fn runner(mut config: Config, rx: Receiver, tx: Sender { + CoreMessage::RetryConnect(retry_idx) => { debug!("Retrying idx: {} (vs. {})", retry_idx, attempt_idx); if retry_idx == attempt_idx { if let Some(progress) = retrying.take() { @@ -110,68 +108,68 @@ async fn runner(mut config: Config, rx: Receiver, tx: Sender { + CoreMessage::Disconnect => { let last_conn = connection.take(); - let _ = interconnect.mixer.send(MixerMessage::DropConn); - let _ = interconnect.mixer.send(MixerMessage::RebuildEncoder); + drop(interconnect.mixer.send(MixerMessage::DropConn)); + drop(interconnect.mixer.send(MixerMessage::RebuildEncoder)); if let Some(conn) = last_conn { - let _ = interconnect.events.send(EventMessage::FireCoreEvent( + drop(interconnect.events.send(EventMessage::FireCoreEvent( CoreContext::DriverDisconnect(InternalDisconnect { kind: DisconnectKind::Runtime, reason: None, info: conn.info.clone(), }), - )); + ))); } }, - Ok(CoreMessage::SignalWsClosure(ws_idx, ws_info, mut reason)) => { + CoreMessage::SignalWsClosure(ws_idx, ws_info, mut reason) => { // if idx is not a match, quash reason // (i.e., prevent users from mistakenly trying to reconnect for an *old* dead conn). // if it *is* a match, the conn needs to die! // (as the WS channel has truly given up the ghost). - if ws_idx != attempt_idx { - reason = None; - } else { + if ws_idx == attempt_idx { connection = None; - let _ = interconnect.mixer.send(MixerMessage::DropConn); - let _ = interconnect.mixer.send(MixerMessage::RebuildEncoder); + drop(interconnect.mixer.send(MixerMessage::DropConn)); + drop(interconnect.mixer.send(MixerMessage::RebuildEncoder)); + } else { + reason = None; } - let _ = interconnect.events.send(EventMessage::FireCoreEvent( + drop(interconnect.events.send(EventMessage::FireCoreEvent( CoreContext::DriverDisconnect(InternalDisconnect { kind: DisconnectKind::Runtime, reason, info: ws_info, }), - )); + ))); }, - Ok(CoreMessage::SetTrack(s)) => { - let _ = interconnect.mixer.send(MixerMessage::SetTrack(s)); + CoreMessage::SetTrack(s) => { + drop(interconnect.mixer.send(MixerMessage::SetTrack(s))); }, - Ok(CoreMessage::AddTrack(s)) => { - let _ = interconnect.mixer.send(MixerMessage::AddTrack(s)); + CoreMessage::AddTrack(s) => { + drop(interconnect.mixer.send(MixerMessage::AddTrack(s))); }, - Ok(CoreMessage::SetBitrate(b)) => { - let _ = interconnect.mixer.send(MixerMessage::SetBitrate(b)); + CoreMessage::SetBitrate(b) => { + drop(interconnect.mixer.send(MixerMessage::SetBitrate(b))); }, - Ok(CoreMessage::SetConfig(mut new_config)) => { + CoreMessage::SetConfig(mut new_config) => { next_config = Some(new_config.clone()); new_config.make_safe(&config, connection.is_some()); - let _ = interconnect.mixer.send(MixerMessage::SetConfig(new_config)); + drop(interconnect.mixer.send(MixerMessage::SetConfig(new_config))); }, - Ok(CoreMessage::AddEvent(evt)) => { - let _ = interconnect.events.send(EventMessage::AddGlobalEvent(evt)); + CoreMessage::AddEvent(evt) => { + drop(interconnect.events.send(EventMessage::AddGlobalEvent(evt))); }, - Ok(CoreMessage::RemoveGlobalEvents) => { - let _ = interconnect.events.send(EventMessage::RemoveGlobalEvents); + CoreMessage::RemoveGlobalEvents => { + drop(interconnect.events.send(EventMessage::RemoveGlobalEvents)); }, - Ok(CoreMessage::Mute(m)) => { - let _ = interconnect.mixer.send(MixerMessage::SetMute(m)); + CoreMessage::Mute(m) => { + drop(interconnect.mixer.send(MixerMessage::SetMute(m))); }, - Ok(CoreMessage::Reconnect) => { + CoreMessage::Reconnect => { if let Some(mut conn) = connection.take() { // try once: if interconnect, try again. // if still issue, full connect. @@ -201,16 +199,16 @@ async fn runner(mut config: Config, rx: Receiver, tx: Sender + CoreMessage::FullReconnect => if let Some(conn) = connection.take() { let info = conn.info.clone(); @@ -218,12 +216,10 @@ async fn runner(mut config: Config, rx: Receiver, tx: Sender { + CoreMessage::RebuildInterconnect => { interconnect.restart_volatile_internals(); }, - Err(RecvError::Disconnected) | Ok(CoreMessage::Poison) => { - break; - }, + CoreMessage::Poison => break, } } @@ -275,22 +271,22 @@ impl ConnectionRetryData { match self.flavour { ConnectionFlavour::Connect(tx) => { // Other side may not be listening: this is fine. - let _ = tx.send(Ok(())); + drop(tx.send(Ok(()))); - let _ = interconnect.events.send(EventMessage::FireCoreEvent( + drop(interconnect.events.send(EventMessage::FireCoreEvent( CoreContext::DriverConnect(InternalConnect { info: connection.info.clone(), ssrc: connection.ssrc, }), - )); + ))); }, ConnectionFlavour::Reconnect => { - let _ = interconnect.events.send(EventMessage::FireCoreEvent( + drop(interconnect.events.send(EventMessage::FireCoreEvent( CoreContext::DriverReconnect(InternalConnect { info: connection.info.clone(), ssrc: connection.ssrc, }), - )); + ))); }, } @@ -304,7 +300,7 @@ impl ConnectionRetryData { spawn(async move { tsleep(t).await; - let _ = remote_ic.core.send(CoreMessage::RetryConnect(idx)); + drop(remote_ic.core.send(CoreMessage::RetryConnect(idx))); }); self.attempts += 1; @@ -325,24 +321,24 @@ impl ConnectionRetryData { match self.flavour { ConnectionFlavour::Connect(tx) => { // See above. - let _ = tx.send(Err(why)); + drop(tx.send(Err(why))); - let _ = interconnect.events.send(EventMessage::FireCoreEvent( + drop(interconnect.events.send(EventMessage::FireCoreEvent( CoreContext::DriverDisconnect(InternalDisconnect { kind: DisconnectKind::Connect, reason, info: self.info, }), - )); + ))); }, ConnectionFlavour::Reconnect => { - let _ = interconnect.events.send(EventMessage::FireCoreEvent( + drop(interconnect.events.send(EventMessage::FireCoreEvent( CoreContext::DriverDisconnect(InternalDisconnect { kind: DisconnectKind::Reconnect, reason, info: self.info, }), - )); + ))); }, } } diff --git a/src/driver/tasks/udp_rx.rs b/src/driver/tasks/udp_rx.rs index 76f5aaa..2d8e830 100644 --- a/src/driver/tasks/udp_rx.rs +++ b/src/driver/tasks/udp_rx.rs @@ -5,7 +5,7 @@ use super::{ }; use crate::{ constants::*, - driver::DecodeMode, + driver::{CryptoMode, DecodeMode}, events::{internal_data::*, CoreContext}, }; use audiopus::{ @@ -53,27 +53,25 @@ enum PacketDecodeSize { impl PacketDecodeSize { fn bump_up(self) -> Self { - use PacketDecodeSize::*; match self { - TwentyMillis => ThirtyMillis, - ThirtyMillis => FortyMillis, - FortyMillis => SixtyMillis, - SixtyMillis | Max => Max, + Self::TwentyMillis => Self::ThirtyMillis, + Self::ThirtyMillis => Self::FortyMillis, + Self::FortyMillis => Self::SixtyMillis, + Self::SixtyMillis | Self::Max => Self::Max, } } fn can_bump_up(self) -> bool { - self != PacketDecodeSize::Max + self != Self::Max } fn len(self) -> usize { - use PacketDecodeSize::*; match self { - TwentyMillis => STEREO_FRAME_SIZE, - ThirtyMillis => (STEREO_FRAME_SIZE / 2) * 3, - FortyMillis => 2 * STEREO_FRAME_SIZE, - SixtyMillis => 3 * STEREO_FRAME_SIZE, - Max => 6 * STEREO_FRAME_SIZE, + Self::TwentyMillis => STEREO_FRAME_SIZE, + Self::ThirtyMillis => (STEREO_FRAME_SIZE / 2) * 3, + Self::FortyMillis => 2 * STEREO_FRAME_SIZE, + Self::SixtyMillis => 3 * STEREO_FRAME_SIZE, + Self::Max => 6 * STEREO_FRAME_SIZE, } } } @@ -86,7 +84,7 @@ enum SpeakingDelta { } impl SsrcState { - fn new(pkt: RtpPacket<'_>) -> Self { + fn new(pkt: &RtpPacket<'_>) -> Self { Self { silent_frame_count: 5, // We do this to make the first speech packet fire an event. decoder: OpusDecoder::new(SAMPLE_RATE, Channels::Stereo) @@ -98,7 +96,7 @@ impl SsrcState { fn process( &mut self, - pkt: RtpPacket<'_>, + pkt: &RtpPacket<'_>, data_offset: usize, data_trailer: usize, decode_mode: DecodeMode, @@ -198,11 +196,10 @@ impl SsrcState { // and then remember that. loop { let tried_audio_len = self.decoder.decode( - Some((&data[start..]).try_into()?), + Some(data[start..].try_into()?), (&mut out[..]).try_into()?, false, ); - match tried_audio_len { Ok(audio_len) => { // Decoding to stereo: audio_len refers to sample count irrespective of channel count. @@ -243,7 +240,6 @@ struct UdpRx { config: Config, packet_buffer: [u8; VOICE_PACKET_MAX], rx: Receiver, - udp_socket: Arc, } @@ -256,15 +252,14 @@ impl UdpRx { self.process_udp_message(interconnect, len); } msg = self.rx.recv_async() => { - use UdpRxMessage::*; match msg { - Ok(ReplaceInterconnect(i)) => { + Ok(UdpRxMessage::ReplaceInterconnect(i)) => { *interconnect = i; }, - Ok(SetConfig(c)) => { + Ok(UdpRxMessage::SetConfig(c)) => { self.config = c; }, - Ok(Poison) | Err(_) => break, + Err(flume::RecvError::Disconnected) => break, } } } @@ -284,7 +279,7 @@ impl UdpRx { match demux::demux_mut(packet) { DemuxedMut::Rtp(mut rtp) => { - if !rtp_valid(rtp.to_immutable()) { + if !rtp_valid(&rtp.to_immutable()) { error!("Illegal RTP message received."); return; } @@ -303,9 +298,10 @@ impl UdpRx { None }; + let rtp = rtp.to_immutable(); let (rtp_body_start, rtp_body_tail, decrypted) = packet_data.unwrap_or_else(|| { ( - crypto_mode.payload_prefix_len(), + CryptoMode::payload_prefix_len(), crypto_mode.payload_suffix_len(), false, ) @@ -314,10 +310,10 @@ impl UdpRx { let entry = self .decoder_map .entry(rtp.get_ssrc()) - .or_insert_with(|| SsrcState::new(rtp.to_immutable())); + .or_insert_with(|| SsrcState::new(&rtp)); if let Ok((delta, audio)) = entry.process( - rtp.to_immutable(), + &rtp, rtp_body_start, rtp_body_tail, self.config.decode_mode, @@ -325,32 +321,32 @@ impl UdpRx { ) { match delta { SpeakingDelta::Start => { - let _ = interconnect.events.send(EventMessage::FireCoreEvent( + drop(interconnect.events.send(EventMessage::FireCoreEvent( CoreContext::SpeakingUpdate(InternalSpeakingUpdate { ssrc: rtp.get_ssrc(), speaking: true, }), - )); + ))); }, SpeakingDelta::Stop => { - let _ = interconnect.events.send(EventMessage::FireCoreEvent( + drop(interconnect.events.send(EventMessage::FireCoreEvent( CoreContext::SpeakingUpdate(InternalSpeakingUpdate { ssrc: rtp.get_ssrc(), speaking: false, }), - )); + ))); }, - _ => {}, + SpeakingDelta::Same => {}, } - let _ = interconnect.events.send(EventMessage::FireCoreEvent( + drop(interconnect.events.send(EventMessage::FireCoreEvent( CoreContext::VoicePacket(InternalVoicePacket { audio, packet: rtp.from_packet(), payload_offset: rtp_body_start, payload_end_pad: rtp_body_tail, }), - )); + ))); } else { warn!("RTP decoding/processing failed."); } @@ -370,26 +366,23 @@ impl UdpRx { let (start, tail) = packet_data.unwrap_or_else(|| { ( - crypto_mode.payload_prefix_len(), + CryptoMode::payload_prefix_len(), crypto_mode.payload_suffix_len(), ) }); - let _ = - interconnect - .events - .send(EventMessage::FireCoreEvent(CoreContext::RtcpPacket( - InternalRtcpPacket { - packet: rtcp.from_packet(), - payload_offset: start, - payload_end_pad: tail, - }, - ))); + drop(interconnect.events.send(EventMessage::FireCoreEvent( + CoreContext::RtcpPacket(InternalRtcpPacket { + packet: rtcp.from_packet(), + payload_offset: start, + payload_end_pad: tail, + }), + ))); }, DemuxedMut::FailedParse(t) => { warn!("Failed to parse message of type {:?}.", t); }, - _ => { + DemuxedMut::TooSmall => { warn!("Illegal UDP packet from voice server."); }, } @@ -408,7 +401,7 @@ pub(crate) async fn runner( let mut state = UdpRx { cipher, - decoder_map: Default::default(), + decoder_map: HashMap::new(), config, packet_buffer: [0u8; VOICE_PACKET_MAX], rx, @@ -421,6 +414,6 @@ pub(crate) async fn runner( } #[inline] -fn rtp_valid(packet: RtpPacket<'_>) -> bool { +fn rtp_valid(packet: &RtpPacket<'_>) -> bool { packet.get_version() == RTP_VERSION && packet.get_payload_type() == RTP_PROFILE_TYPE } diff --git a/src/driver/tasks/udp_tx.rs b/src/driver/tasks/udp_tx.rs index 8e482d4..7eb9e85 100644 --- a/src/driver/tasks/udp_tx.rs +++ b/src/driver/tasks/udp_tx.rs @@ -12,7 +12,6 @@ use tracing::{error, instrument, trace}; struct UdpTx { ssrc: u32, rx: Receiver, - udp_tx: Arc, } @@ -26,7 +25,6 @@ impl UdpTx { let mut ka_time = Instant::now() + UDP_KEEPALIVE_GAP; loop { - use UdpTxMessage::*; match timeout_at(ka_time, self.rx.recv_async()).await { Err(_) => { trace!("Sending UDP Keepalive."); @@ -36,16 +34,12 @@ impl UdpTx { } ka_time += UDP_KEEPALIVE_GAP; }, - Ok(Ok(Packet(p))) => + Ok(Ok(p)) => if let Err(e) = self.udp_tx.send(&p[..]).await { error!("Fatal UDP packet send error: {:?}.", e); break; }, - Ok(Err(e)) => { - error!("Fatal UDP packet receive error: {:?}.", e); - break; - }, - Ok(Ok(Poison)) => { + Ok(Err(flume::RecvError::Disconnected)) => { break; }, } diff --git a/src/driver/tasks/ws.rs b/src/driver/tasks/ws.rs index 65d2763..c7a3626 100644 --- a/src/driver/tasks/ws.rs +++ b/src/driver/tasks/ws.rs @@ -140,7 +140,7 @@ impl AuxNetwork { } } }, - Err(_) | Ok(WsMessage::Poison) => { + Err(flume::RecvError::Disconnected) => { break; }, } @@ -151,13 +151,13 @@ impl AuxNetwork { self.dont_send = true; if should_reconnect { - let _ = interconnect.core.send(CoreMessage::Reconnect); + drop(interconnect.core.send(CoreMessage::Reconnect)); } else { - let _ = interconnect.core.send(CoreMessage::SignalWsClosure( + drop(interconnect.core.send(CoreMessage::SignalWsClosure( self.attempt_idx, self.info.clone(), ws_reason, - )); + ))); break; } } @@ -186,17 +186,17 @@ impl AuxNetwork { fn process_ws(&mut self, interconnect: &Interconnect, value: GatewayEvent) { match value { GatewayEvent::Speaking(ev) => { - let _ = interconnect.events.send(EventMessage::FireCoreEvent( + drop(interconnect.events.send(EventMessage::FireCoreEvent( CoreContext::SpeakingStateUpdate(ev), - )); + ))); }, GatewayEvent::ClientConnect(ev) => { debug!("Received discontinued ClientConnect: {:?}", ev); }, GatewayEvent::ClientDisconnect(ev) => { - let _ = interconnect.events.send(EventMessage::FireCoreEvent( + drop(interconnect.events.send(EventMessage::FireCoreEvent( CoreContext::ClientDisconnect(ev), - )); + ))); }, GatewayEvent::HeartbeatAck(ev) => { if let Some(nonce) = self.last_heartbeat_nonce.take() { diff --git a/src/driver/test_config.rs b/src/driver/test_config.rs new file mode 100644 index 0000000..0536561 --- /dev/null +++ b/src/driver/test_config.rs @@ -0,0 +1,251 @@ +#![allow(missing_docs)] + +use flume::{Receiver, Sender}; + +use crate::{ + tracks::{PlayMode, TrackHandle, TrackState}, + Event, + EventContext, + EventHandler, + TrackEvent, +}; +use std::time::Duration; + +#[allow(dead_code)] +#[derive(Clone, Debug)] +pub enum TickStyle { + Timed, + UntimedWithExecLimit(Receiver), +} + +#[derive(Clone, Debug, PartialEq)] +pub enum OutputMessage { + Passthrough(Vec), + Mixed(Vec), + Silent, +} + +#[allow(dead_code)] +impl OutputMessage { + pub fn is_passthrough(&self) -> bool { + matches!(self, Self::Passthrough(_)) + } + + pub fn is_mixed(&self) -> bool { + matches!(self, Self::Mixed(_)) + } + + pub fn is_mixed_with_nonzero_signal(&self) -> bool { + if let Self::Mixed(data) = self { + data.iter().any(|v| *v != 0.0f32) + } else { + false + } + } + + pub fn is_explicit_silence(&self) -> bool { + *self == Self::Silent + } +} + +#[allow(dead_code)] +#[derive(Clone, Debug)] +pub enum OutputMode { + Raw(Sender>), + Rtp(Sender>>), +} + +#[allow(dead_code)] +#[derive(Clone, Debug)] +pub enum TickMessage { + El(T), + NoEl, +} + +impl From for TickMessage { + fn from(val: T) -> Self { + TickMessage::El(val) + } +} + +impl From> for OutputPacket { + fn from(val: TickMessage) -> Self { + match val { + TickMessage::El(e) => OutputPacket::Raw(e), + TickMessage::NoEl => OutputPacket::Empty, + } + } +} + +impl From>> for OutputPacket { + fn from(val: TickMessage>) -> Self { + match val { + TickMessage::El(e) => OutputPacket::Rtp(e), + TickMessage::NoEl => OutputPacket::Empty, + } + } +} + +#[derive(Clone, Debug, PartialEq)] +pub enum OutputPacket { + Raw(OutputMessage), + Rtp(Vec), + Empty, +} + +impl OutputPacket { + pub fn raw(&self) -> Option<&OutputMessage> { + if let Self::Raw(o) = self { + Some(o) + } else { + None + } + } +} + +#[derive(Clone, Debug)] +pub enum OutputReceiver { + Raw(Receiver>), + Rtp(Receiver>>), +} + +#[derive(Clone)] +pub struct DriverTestHandle { + pub rx: OutputReceiver, + pub tx: Sender, +} + +impl DriverTestHandle { + pub fn recv(&self) -> OutputPacket { + match &self.rx { + OutputReceiver::Raw(rx) => rx.recv().unwrap().into(), + OutputReceiver::Rtp(rx) => rx.recv().unwrap().into(), + } + } + + pub async fn recv_async(&self) -> OutputPacket { + match &self.rx { + OutputReceiver::Raw(rx) => rx.recv_async().await.unwrap().into(), + OutputReceiver::Rtp(rx) => rx.recv_async().await.unwrap().into(), + } + } + + pub fn len(&self) -> usize { + match &self.rx { + OutputReceiver::Raw(rx) => rx.len(), + OutputReceiver::Rtp(rx) => rx.len(), + } + } + + pub fn wait(&self, n_ticks: u64) { + for _i in 0..n_ticks { + drop(self.recv()); + } + } + + pub async fn wait_async(&self, n_ticks: u64) { + for _i in 0..n_ticks { + drop(self.recv_async().await); + } + } + + pub async fn spawn_ticker(&self) { + let remote = self.clone(); + tokio::spawn(async move { + loop { + remote.skip(1).await; + tokio::time::sleep(Duration::from_millis(1)).await; + } + }); + } + + pub fn wait_noisy(&self, n_ticks: u64) { + for _i in 0..n_ticks { + match self.recv() { + OutputPacket::Empty => eprintln!("pkt: Nothing"), + OutputPacket::Rtp(p) => eprintln!("pkt: RTP[{}B]", p.len()), + OutputPacket::Raw(OutputMessage::Silent) => eprintln!("pkt: Raw-Silent"), + OutputPacket::Raw(OutputMessage::Passthrough(p)) => + eprintln!("pkt: Raw-Passthrough[{}B]", p.len()), + OutputPacket::Raw(OutputMessage::Mixed(p)) => + eprintln!("pkt: Raw-Mixed[{}B]", p.len()), + } + } + } + + pub async fn skip(&self, n_ticks: u64) { + self.tick(n_ticks); + self.wait_async(n_ticks).await; + } + + pub fn tick(&self, n_ticks: u64) { + if n_ticks == 0 { + panic!("Number of ticks to advance driver/mixer must be >= 1."); + } + self.tx.send(n_ticks).unwrap(); + } + + pub async fn ready_track( + &self, + handle: &TrackHandle, + tick_wait: Option, + ) -> TrackState { + let (tx, rx) = flume::bounded(1); + let (err_tx, err_rx) = flume::bounded(1); + + struct SongPlayable { + tx: Sender, + } + + #[async_trait::async_trait] + impl EventHandler for SongPlayable { + async fn act(&self, ctx: &crate::EventContext<'_>) -> Option { + if let EventContext::Track(&[(state, _)]) = ctx { + drop(self.tx.send(state.clone())); + } + + Some(Event::Cancel) + } + } + + struct SongErred { + tx: Sender, + } + + #[async_trait::async_trait] + impl EventHandler for SongErred { + async fn act(&self, ctx: &crate::EventContext<'_>) -> Option { + if let EventContext::Track(&[(state, _)]) = ctx { + drop(self.tx.send(state.playing.clone())); + } + + Some(Event::Cancel) + } + } + + handle + .add_event(Event::Track(TrackEvent::Playable), SongPlayable { tx }) + .expect("Adding track evt should not fail before any ticks."); + + handle + .add_event(Event::Track(TrackEvent::Error), SongErred { tx: err_tx }) + .expect("Adding track evt should not fail before any ticks."); + + loop { + self.tick(1); + tokio::time::sleep(tick_wait.unwrap_or_else(|| Duration::from_millis(20))).await; + self.wait_async(1).await; + + match err_rx.try_recv() { + Ok(e) => panic!("Error reported on track: {:?}", e), + Err(flume::TryRecvError::Empty | flume::TryRecvError::Disconnected) => {}, + } + + match rx.try_recv() { + Ok(val) => return val, + Err(flume::TryRecvError::Disconnected) => panic!(), + Err(flume::TryRecvError::Empty) => {}, + } + } + } +} diff --git a/src/error.rs b/src/error.rs index 9a8520c..2691f4e 100644 --- a/src/error.rs +++ b/src/error.rs @@ -4,12 +4,12 @@ use futures::channel::mpsc::TrySendError; #[cfg(feature = "serenity")] use serenity::gateway::InterMessage; -#[cfg(feature = "gateway-core")] +#[cfg(feature = "gateway")] use std::{error::Error, fmt}; #[cfg(feature = "twilight")] use twilight_gateway::{cluster::ClusterCommandError, shard::CommandError}; -#[cfg(feature = "gateway-core")] +#[cfg(feature = "gateway")] #[derive(Debug)] #[non_exhaustive] /// Error returned when a manager or call handler is @@ -36,11 +36,7 @@ pub enum JoinError { /// /// [the `Call`'s configuration]: crate::Config TimedOut, - /// The given guild ID was zero. - IllegalGuild, - /// The given channel ID was zero. - IllegalChannel, - #[cfg(feature = "driver-core")] + #[cfg(feature = "driver")] /// The driver failed to establish a voice connection. /// /// *Users should `leave` the server on the gateway before @@ -57,7 +53,7 @@ pub enum JoinError { TwilightShard(CommandError), } -#[cfg(feature = "gateway-core")] +#[cfg(feature = "gateway")] impl JoinError { /// Indicates whether this failure may have left (or been /// caused by) Discord's gateway state being in an @@ -69,7 +65,7 @@ impl JoinError { matches!(self, JoinError::TimedOut) } - #[cfg(feature = "driver-core")] + #[cfg(feature = "driver")] /// Indicates whether this failure can be reattempted via /// [`Driver::connect`] with retreived connection info. /// @@ -82,7 +78,7 @@ impl JoinError { } } -#[cfg(feature = "gateway-core")] +#[cfg(feature = "gateway")] impl fmt::Display for JoinError { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "failed to join voice channel: ")?; @@ -91,9 +87,7 @@ impl fmt::Display for JoinError { JoinError::NoSender => write!(f, "no gateway destination"), JoinError::NoCall => write!(f, "tried to leave a non-existent call"), JoinError::TimedOut => write!(f, "gateway response from Discord timed out"), - JoinError::IllegalGuild => write!(f, "target guild ID was zero"), - JoinError::IllegalChannel => write!(f, "target channel ID was zero"), - #[cfg(feature = "driver-core")] + #[cfg(feature = "driver")] JoinError::Driver(_) => write!(f, "establishing connection failed"), #[cfg(feature = "serenity")] JoinError::Serenity(e) => e.fmt(f), @@ -105,7 +99,7 @@ impl fmt::Display for JoinError { } } -#[cfg(feature = "gateway-core")] +#[cfg(feature = "gateway")] impl Error for JoinError { fn source(&self) -> Option<&(dyn Error + 'static)> { match self { @@ -113,9 +107,7 @@ impl Error for JoinError { JoinError::NoSender => None, JoinError::NoCall => None, JoinError::TimedOut => None, - JoinError::IllegalGuild => None, - JoinError::IllegalChannel => None, - #[cfg(feature = "driver-core")] + #[cfg(feature = "driver")] JoinError::Driver(e) => Some(e), #[cfg(feature = "serenity")] JoinError::Serenity(e) => e.source(), @@ -127,40 +119,40 @@ impl Error for JoinError { } } -#[cfg(all(feature = "serenity", feature = "gateway-core"))] +#[cfg(all(feature = "serenity", feature = "gateway"))] impl From> for JoinError { fn from(e: TrySendError) -> Self { JoinError::Serenity(e) } } -#[cfg(all(feature = "twilight", feature = "gateway-core"))] +#[cfg(all(feature = "twilight", feature = "gateway"))] impl From for JoinError { fn from(e: CommandError) -> Self { JoinError::TwilightShard(e) } } -#[cfg(all(feature = "twilight", feature = "gateway-core"))] +#[cfg(all(feature = "twilight", feature = "gateway"))] impl From for JoinError { fn from(e: ClusterCommandError) -> Self { JoinError::TwilightCluster(e) } } -#[cfg(all(feature = "driver-core", feature = "gateway-core"))] +#[cfg(all(feature = "driver", feature = "gateway"))] impl From for JoinError { fn from(e: ConnectionError) -> Self { JoinError::Driver(e) } } -#[cfg(feature = "gateway-core")] +#[cfg(feature = "gateway")] /// Convenience type for Discord gateway error handling. pub type JoinResult = Result; -#[cfg(feature = "driver-core")] +#[cfg(feature = "driver")] pub use crate::{ driver::connection::error::{Error as ConnectionError, Result as ConnectionResult}, - tracks::{TrackError, TrackResult}, + tracks::{ControlError, PlayError, TrackResult}, }; diff --git a/src/events/context/data/disconnect.rs b/src/events/context/data/disconnect.rs index 89e49bc..6fbbba0 100644 --- a/src/events/context/data/disconnect.rs +++ b/src/events/context/data/disconnect.rs @@ -84,21 +84,19 @@ pub enum DisconnectReason { impl From<&ConnectionError> for DisconnectReason { fn from(e: &ConnectionError) -> Self { - use ConnectionError::*; - match e { - AttemptDiscarded => Self::AttemptDiscarded, - CryptoModeInvalid - | CryptoModeUnavailable - | EndpointUrl - | ExpectedHandshake - | IllegalDiscoveryResponse - | IllegalIp - | Json(_) => Self::ProtocolViolation, - Io(_) => Self::Io, - Crypto(_) | InterconnectFailure(_) => Self::Internal, - Ws(ws) => ws.into(), - TimedOut => Self::TimedOut, + ConnectionError::AttemptDiscarded => Self::AttemptDiscarded, + ConnectionError::CryptoModeInvalid + | ConnectionError::CryptoModeUnavailable + | ConnectionError::EndpointUrl + | ConnectionError::ExpectedHandshake + | ConnectionError::IllegalDiscoveryResponse + | ConnectionError::IllegalIp + | ConnectionError::Json(_) => Self::ProtocolViolation, + ConnectionError::Io(_) => Self::Io, + ConnectionError::Crypto(_) | ConnectionError::InterconnectFailure(_) => Self::Internal, + ConnectionError::Ws(ws) => ws.into(), + ConnectionError::TimedOut => Self::TimedOut, } } } diff --git a/src/events/context/mod.rs b/src/events/context/mod.rs index 98d2e5c..693dc03 100644 --- a/src/events/context/mod.rs +++ b/src/events/context/mod.rs @@ -62,17 +62,17 @@ pub enum CoreContext { impl<'a> CoreContext { pub(crate) fn to_user_context(&'a self) -> EventContext<'a> { - use CoreContext::*; - match self { - SpeakingStateUpdate(evt) => EventContext::SpeakingStateUpdate(*evt), - SpeakingUpdate(evt) => EventContext::SpeakingUpdate(SpeakingUpdateData::from(evt)), - VoicePacket(evt) => EventContext::VoicePacket(VoiceData::from(evt)), - RtcpPacket(evt) => EventContext::RtcpPacket(RtcpData::from(evt)), - ClientDisconnect(evt) => EventContext::ClientDisconnect(*evt), - DriverConnect(evt) => EventContext::DriverConnect(ConnectData::from(evt)), - DriverReconnect(evt) => EventContext::DriverReconnect(ConnectData::from(evt)), - DriverDisconnect(evt) => EventContext::DriverDisconnect(DisconnectData::from(evt)), + Self::SpeakingStateUpdate(evt) => EventContext::SpeakingStateUpdate(*evt), + Self::SpeakingUpdate(evt) => + EventContext::SpeakingUpdate(SpeakingUpdateData::from(evt)), + Self::VoicePacket(evt) => EventContext::VoicePacket(VoiceData::from(evt)), + Self::RtcpPacket(evt) => EventContext::RtcpPacket(RtcpData::from(evt)), + Self::ClientDisconnect(evt) => EventContext::ClientDisconnect(*evt), + Self::DriverConnect(evt) => EventContext::DriverConnect(ConnectData::from(evt)), + Self::DriverReconnect(evt) => EventContext::DriverReconnect(ConnectData::from(evt)), + Self::DriverDisconnect(evt) => + EventContext::DriverDisconnect(DisconnectData::from(evt)), } } } @@ -80,18 +80,17 @@ impl<'a> CoreContext { impl EventContext<'_> { /// Retreive the event class for an event (i.e., when matching) /// an event against the registered listeners. + #[must_use] pub fn to_core_event(&self) -> Option { - use EventContext::*; - match self { - SpeakingStateUpdate(_) => Some(CoreEvent::SpeakingStateUpdate), - SpeakingUpdate(_) => Some(CoreEvent::SpeakingUpdate), - VoicePacket(_) => Some(CoreEvent::VoicePacket), - RtcpPacket(_) => Some(CoreEvent::RtcpPacket), - ClientDisconnect(_) => Some(CoreEvent::ClientDisconnect), - DriverConnect(_) => Some(CoreEvent::DriverConnect), - DriverReconnect(_) => Some(CoreEvent::DriverReconnect), - DriverDisconnect(_) => Some(CoreEvent::DriverDisconnect), + Self::SpeakingStateUpdate(_) => Some(CoreEvent::SpeakingStateUpdate), + Self::SpeakingUpdate(_) => Some(CoreEvent::SpeakingUpdate), + Self::VoicePacket(_) => Some(CoreEvent::VoicePacket), + Self::RtcpPacket(_) => Some(CoreEvent::RtcpPacket), + Self::ClientDisconnect(_) => Some(CoreEvent::ClientDisconnect), + Self::DriverConnect(_) => Some(CoreEvent::DriverConnect), + Self::DriverReconnect(_) => Some(CoreEvent::DriverReconnect), + Self::DriverDisconnect(_) => Some(CoreEvent::DriverDisconnect), _ => None, } } diff --git a/src/events/core.rs b/src/events/core.rs index f88e440..b73ac0b 100644 --- a/src/events/core.rs +++ b/src/events/core.rs @@ -9,7 +9,7 @@ /// when a client leaves the session ([`ClientDisconnect`]), voice packets ([`VoicePacket`]), and /// telemetry data ([`RtcpPacket`]). The format of voice packets is described by [`VoiceData`]. /// -/// To detect when a user connects, you must correlate gateway (e.g., VoiceStateUpdate) events +/// To detect when a user connects, you must correlate gateway (e.g., `VoiceStateUpdate`) events /// from the main part of your bot. /// /// To obtain a user's SSRC, you must use [`SpeakingStateUpdate`] events. diff --git a/src/events/store.rs b/src/events/store.rs index 7d5ccbf..6d57dd8 100644 --- a/src/events/store.rs +++ b/src/events/store.rs @@ -1,7 +1,7 @@ use super::*; use crate::{ constants::*, - tracks::{PlayMode, TrackHandle, TrackState}, + tracks::{ReadyState, TrackHandle, TrackState}, }; use std::{ collections::{BinaryHeap, HashMap}, @@ -24,8 +24,9 @@ pub struct EventStore { impl EventStore { /// Creates a new event store to be used globally. + #[must_use] pub fn new() -> Self { - Default::default() + Self::default() } /// Creates a new event store to be used within a [`Track`]. @@ -34,6 +35,7 @@ impl EventStore { /// a track has been registered. /// /// [`Track`]: crate::tracks::Track + #[must_use] pub fn new_local() -> Self { EventStore { local_only: true, @@ -53,21 +55,20 @@ impl EventStore { return; } - use Event::*; match evt.event { - Core(c) => { + Event::Core(c) => { self.untimed .entry(c.into()) .or_insert_with(Vec::new) .push(evt); }, - Track(t) => { + Event::Track(t) => { self.untimed .entry(t.into()) .or_insert_with(Vec::new) .push(evt); }, - Delayed(_) | Periodic(_, _) => { + Event::Delayed(_) | Event::Periodic(_, _) => { self.timed.push(evt); }, _ => { @@ -105,15 +106,12 @@ impl EventStore { /// Processes all events due up to and including `now`. pub(crate) fn timed_event_ready(&self, now: Duration) -> bool { - self.timed - .peek() - .map(|evt| { - evt.fire_time - .as_ref() - .expect("Timed event must have a fire_time.") - <= &now - }) - .unwrap_or(false) + self.timed.peek().map_or(false, |evt| { + evt.fire_time + .as_ref() + .expect("Timed event must have a fire_time.") + <= &now + }) } /// Processes all events attached to the given track event. @@ -183,9 +181,9 @@ impl GlobalEvents { pub(crate) async fn tick( &mut self, - events: &mut Vec, - states: &mut Vec, - handles: &mut Vec, + events: &mut [EventStore], + states: &mut [TrackState], + handles: &mut [TrackHandle], ) { // Global timed events self.time += TIMESTEP_LENGTH; @@ -199,7 +197,7 @@ impl GlobalEvents { // Local timed events for (i, state) in states.iter_mut().enumerate() { - if state.playing == PlayMode::Play { + if state.playing.is_playing() && state.ready == ReadyState::Playable { state.step_frame(); let event_store = events @@ -215,7 +213,7 @@ impl GlobalEvents { } } - for (evt, indices) in self.awaiting_tick.iter() { + for (evt, indices) in &self.awaiting_tick { let untimed = (*evt).into(); if !indices.is_empty() { @@ -223,7 +221,7 @@ impl GlobalEvents { } // Local untimed track events. - for &i in indices.iter() { + for &i in indices { let event_store = events .get_mut(i) .expect("Missing store index for Tick (local untimed)."); @@ -261,12 +259,12 @@ impl GlobalEvents { self.store .process_untimed(self.time, untimed, EventContext::Track(&global_ctx[..])) - .await + .await; } } // Now drain vecs. - for (_evt, indices) in self.awaiting_tick.iter_mut() { + for indices in self.awaiting_tick.values_mut() { indices.clear(); } } diff --git a/src/events/track.rs b/src/events/track.rs index c0b6be4..8660a90 100644 --- a/src/events/track.rs +++ b/src/events/track.rs @@ -1,3 +1,6 @@ +// TODO: Could this be a bitset? Could accelerate lookups, +// allow easy joint subscription & remove Vecs for related evt handling? + /// Track events correspond to certain actions or changes /// of state, such as a track finishing, looping, or being /// manually stopped. Voice core events occur on receipt of @@ -14,7 +17,8 @@ pub enum TrackEvent { /// /// This event will not fire when a track first starts, /// but will fire when a track changes from, e.g., paused to playing. - /// This is most relevant for queue users. + /// This is most relevant for queue users: queued tracks placed into a + /// non-empty queue are initlally paused, and are later moved to `Play`. Play, /// The attached track has been paused. Pause, @@ -22,4 +26,10 @@ pub enum TrackEvent { End, /// The attached track has looped. Loop, + /// The attached track is being readied or recreated. + Preparing, + /// The attached track has become playable. + Playable, + /// The attached track has encountered a runtime or initialisation error. + Error, } diff --git a/src/handler.rs b/src/handler.rs index 6f7959a..68bd2ab 100644 --- a/src/handler.rs +++ b/src/handler.rs @@ -1,4 +1,4 @@ -#[cfg(feature = "driver-core")] +#[cfg(feature = "driver")] use crate::{driver::Driver, error::ConnectionResult}; use crate::{ error::{JoinError, JoinResult}, @@ -12,7 +12,7 @@ use flume::Sender; use std::fmt::Debug; use tracing::instrument; -#[cfg(feature = "driver-core")] +#[cfg(feature = "driver")] use std::ops::{Deref, DerefMut}; #[derive(Clone, Debug)] @@ -24,7 +24,7 @@ enum Return { // second indicates that the driver successfully connected. // The first is needed to cancel a timeout as the driver can/should // have separate connection timing/retry config. - #[cfg(feature = "driver-core")] + #[cfg(feature = "driver")] Conn(Sender<()>, Sender>), } @@ -37,12 +37,12 @@ enum Return { /// [`Driver`]: struct@Driver #[derive(Clone, Debug)] pub struct Call { - #[cfg(not(feature = "driver-core"))] + #[cfg(not(feature = "driver"))] config: Config, connection: Option<(ConnectionProgress, Return)>, - #[cfg(feature = "driver-core")] + #[cfg(feature = "driver")] /// The internal controller of the voice connection monitor thread. driver: Driver, @@ -73,12 +73,7 @@ impl Call { G: Into + Debug, U: Into + Debug, { - Self::new_raw_cfg( - guild_id.into(), - Some(ws), - user_id.into(), - Default::default(), - ) + Self::new_raw_cfg(guild_id.into(), Some(ws), user_id.into(), Config::default()) } /// Creates a new Call, configuring the driver as specified. @@ -107,7 +102,7 @@ impl Call { G: Into + Debug, U: Into + Debug, { - Self::new_raw_cfg(guild_id.into(), None, user_id.into(), Default::default()) + Self::new_raw_cfg(guild_id.into(), None, user_id.into(), Config::default()) } /// Creates a new standalone Call from the given configuration file. @@ -123,10 +118,10 @@ impl Call { fn new_raw_cfg(guild_id: GuildId, ws: Option, user_id: UserId, config: Config) -> Self { Call { - #[cfg(not(feature = "driver-core"))] + #[cfg(not(feature = "driver"))] config, connection: None, - #[cfg(feature = "driver-core")] + #[cfg(feature = "driver")] driver: Driver::new(config), guild_id, self_deaf: false, @@ -141,9 +136,9 @@ impl Call { match &self.connection { Some((ConnectionProgress::Complete(c), Return::Info(tx))) => { // It's okay if the receiver hung up. - let _ = tx.send(c.clone()); + drop(tx.send(c.clone())); }, - #[cfg(feature = "driver-core")] + #[cfg(feature = "driver")] Some((ConnectionProgress::Complete(c), Return::Conn(first_tx, driver_tx))) => { // It's okay if the receiver hung up. let _ = first_tx.send(()); @@ -195,7 +190,7 @@ impl Call { self.leave().await?; true } else if conn.0.channel_id() == channel_id { - let _ = tx.send(completion_generator(self)); + drop(tx.send(completion_generator(self))); false } else { // not in progress, and/or a channel change. @@ -206,7 +201,7 @@ impl Call { }) } - #[cfg(feature = "driver-core")] + #[cfg(feature = "driver")] /// Connect or switch to the given voice channel by its Id. /// /// This function acts as a future in two stages: @@ -227,7 +222,7 @@ impl Call { self._join(channel_id.into()).await } - #[cfg(feature = "driver-core")] + #[cfg(feature = "driver")] async fn _join(&mut self, channel_id: ChannelId) -> JoinResult { let (tx, rx) = flume::unbounded(); let (gw_tx, gw_rx) = flume::unbounded(); @@ -359,7 +354,7 @@ impl Call { fn leave_local(&mut self) { self.connection = None; - #[cfg(feature = "driver-core")] + #[cfg(feature = "driver")] self.driver.leave(); } @@ -376,7 +371,7 @@ impl Call { pub async fn mute(&mut self, mute: bool) -> JoinResult<()> { self.self_mute = mute; - #[cfg(feature = "driver-core")] + #[cfg(feature = "driver")] self.driver.mute(mute); self.update().await @@ -419,7 +414,7 @@ impl Call { where C: Into + Debug, { - self._update_state(session_id, channel_id.map(|c| c.into())) + self._update_state(session_id, channel_id.map(Into::into)); } fn _update_state(&mut self, session_id: String, channel_id: Option) { @@ -460,7 +455,7 @@ impl Call { } } -#[cfg(not(feature = "driver-core"))] +#[cfg(not(feature = "driver"))] impl Call { /// Access this call handler's configuration. pub fn config(&self) -> &Config { @@ -478,7 +473,7 @@ impl Call { } } -#[cfg(feature = "driver-core")] +#[cfg(feature = "driver")] impl Deref for Call { type Target = Driver; @@ -487,7 +482,7 @@ impl Deref for Call { } } -#[cfg(feature = "driver-core")] +#[cfg(feature = "driver")] impl DerefMut for Call { fn deref_mut(&mut self) -> &mut Self::Target { &mut self.driver diff --git a/src/id.rs b/src/id.rs index 58ab104..eba6e7f 100644 --- a/src/id.rs +++ b/src/id.rs @@ -1,6 +1,6 @@ //! Newtypes around Discord IDs for library cross-compatibility. -#[cfg(feature = "driver-core")] +#[cfg(feature = "driver")] use crate::model::id::{GuildId as DriverGuild, UserId as DriverUser}; #[cfg(feature = "serenity")] use serenity::model::id::{ @@ -8,7 +8,10 @@ use serenity::model::id::{ GuildId as SerenityGuild, UserId as SerenityUser, }; -use std::fmt::{Display, Formatter, Result as FmtResult}; +use std::{ + fmt::{Display, Formatter, Result as FmtResult}, + num::NonZeroU64, +}; #[cfg(feature = "twilight")] use twilight_model::id::{ marker::{ChannelMarker, GuildMarker, UserMarker}, @@ -16,16 +19,16 @@ use twilight_model::id::{ }; /// ID of a Discord voice/text channel. -#[derive(Clone, Copy, Debug, Default, Eq, Hash, PartialEq)] -pub struct ChannelId(pub u64); +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub struct ChannelId(pub NonZeroU64); /// ID of a Discord guild (colloquially, "server"). -#[derive(Clone, Copy, Debug, Default, Eq, Hash, PartialEq)] -pub struct GuildId(pub u64); +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub struct GuildId(pub NonZeroU64); /// ID of a Discord user. -#[derive(Clone, Copy, Debug, Default, Eq, Hash, PartialEq)] -pub struct UserId(pub u64); +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub struct UserId(pub NonZeroU64); impl Display for ChannelId { fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult { @@ -33,8 +36,8 @@ impl Display for ChannelId { } } -impl From for ChannelId { - fn from(id: u64) -> Self { +impl From for ChannelId { + fn from(id: NonZeroU64) -> Self { Self(id) } } @@ -49,7 +52,7 @@ impl From for ChannelId { #[cfg(feature = "twilight")] impl From> for ChannelId { fn from(id: TwilightId) -> Self { - Self(id.get().into()) + Self(id.into_nonzero()) } } @@ -59,8 +62,8 @@ impl Display for GuildId { } } -impl From for GuildId { - fn from(id: u64) -> Self { +impl From for GuildId { + fn from(id: NonZeroU64) -> Self { Self(id) } } @@ -72,17 +75,17 @@ impl From for GuildId { } } -#[cfg(feature = "driver-core")] +#[cfg(feature = "driver")] impl From for DriverGuild { fn from(id: GuildId) -> Self { - Self(id.0) + Self(id.0.get()) } } #[cfg(feature = "twilight")] impl From> for GuildId { fn from(id: TwilightId) -> Self { - Self(id.get().into()) + Self(id.into_nonzero()) } } @@ -92,8 +95,8 @@ impl Display for UserId { } } -impl From for UserId { - fn from(id: u64) -> Self { +impl From for UserId { + fn from(id: NonZeroU64) -> Self { Self(id) } } @@ -105,16 +108,16 @@ impl From for UserId { } } -#[cfg(feature = "driver-core")] +#[cfg(feature = "driver")] impl From for DriverUser { fn from(id: UserId) -> Self { - Self(id.0) + Self(id.0.get()) } } #[cfg(feature = "twilight")] impl From> for UserId { fn from(id: TwilightId) -> Self { - Self(id.get().into()) + Self(id.into_nonzero()) } } diff --git a/src/info.rs b/src/info.rs index 15e564f..e2f68af 100644 --- a/src/info.rs +++ b/src/info.rs @@ -13,15 +13,17 @@ impl ConnectionProgress { channel_id, guild_id, user_id, - ..Default::default() + token: None, + endpoint: None, + session_id: None, }) } pub(crate) fn get_connection_info(&self) -> Option<&ConnectionInfo> { - use ConnectionProgress::*; - match self { - Complete(c) => Some(c), - _ => None, + if let Self::Complete(c) = self { + Some(c) + } else { + None } } @@ -53,10 +55,7 @@ impl ConnectionProgress { } pub(crate) fn info(&self) -> Option { - match self { - ConnectionProgress::Complete(conn_info) => Some(conn_info.clone()), - _ => None, - } + self.get_connection_info().cloned() } pub(crate) fn apply_state_update(&mut self, session_id: String, channel_id: ChannelId) -> bool { @@ -65,26 +64,24 @@ impl ConnectionProgress { *self = ConnectionProgress::new(self.guild_id(), self.user_id(), channel_id); } - use ConnectionProgress::*; match self { - Complete(c) => { + Self::Complete(c) => { let should_reconn = c.session_id != session_id; c.session_id = session_id; should_reconn }, - Incomplete(i) => i + Self::Incomplete(i) => i .apply_state_update(session_id, channel_id) .map(|info| { - *self = Complete(info); + *self = Self::Complete(info); }) .is_some(), } } pub(crate) fn apply_server_update(&mut self, endpoint: String, token: String) -> bool { - use ConnectionProgress::*; match self { - Complete(c) => { + Self::Complete(c) => { let should_reconn = c.endpoint != endpoint || c.token != token; c.endpoint = endpoint; @@ -92,10 +89,10 @@ impl ConnectionProgress { should_reconn }, - Incomplete(i) => i + Self::Incomplete(i) => i .apply_server_update(endpoint, token) .map(|info| { - *self = Complete(info); + *self = Self::Complete(info); }) .is_some(), } @@ -138,7 +135,7 @@ impl fmt::Debug for ConnectionInfo { } } -#[derive(Clone, Default)] +#[derive(Clone)] pub(crate) struct Partial { pub channel_id: ChannelId, pub endpoint: Option, diff --git a/src/input/adapters/async_adapter.rs b/src/input/adapters/async_adapter.rs new file mode 100644 index 0000000..54d3dfb --- /dev/null +++ b/src/input/adapters/async_adapter.rs @@ -0,0 +1,332 @@ +use crate::input::AudioStreamError; +use async_trait::async_trait; +use flume::{Receiver, RecvError, Sender, TryRecvError}; +use futures::{future::Either, stream::FuturesUnordered, FutureExt, StreamExt}; +use ringbuf::*; +use std::{ + io::{ + Error as IoError, + ErrorKind as IoErrorKind, + Read, + Result as IoResult, + Seek, + SeekFrom, + Write, + }, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, +}; +use symphonia_core::io::MediaSource; +use tokio::{ + io::{AsyncRead, AsyncReadExt, AsyncSeek, AsyncSeekExt}, + sync::Notify, +}; + +struct AsyncAdapterSink { + bytes_in: Producer, + req_rx: Receiver, + resp_tx: Sender, + stream: Box, + notify_rx: Arc, +} + +impl AsyncAdapterSink { + async fn launch(mut self) { + let mut inner_buf = [0u8; 10 * 1024]; + let mut read_region = 0..0; + let mut hit_end = false; + let mut blocked = false; + let mut pause_buf_moves = false; + let mut seek_res = None; + let mut seen_bytes = 0; + + loop { + // if read_region is empty, refill from src. + // if that read is zero, tell other half. + // if WouldBlock, block on msg acquire, + // else non_block msg acquire. + + if !pause_buf_moves { + if !hit_end && read_region.is_empty() { + if let Ok(n) = self.stream.read(&mut inner_buf).await { + read_region = 0..n; + if n == 0 { + drop(self.resp_tx.send_async(AdapterResponse::ReadZero).await); + hit_end = true; + } + seen_bytes += n as u64; + } else { + match self.stream.try_resume(seen_bytes).await { + Ok(s) => { + self.stream = s; + }, + Err(_e) => break, + } + } + } + + while !read_region.is_empty() && !blocked { + if let Ok(n_moved) = self + .bytes_in + .write(&inner_buf[read_region.start..read_region.end]) + { + read_region.start += n_moved; + } else { + blocked = true; + } + } + } + + let msg = if blocked || hit_end { + let mut fs = FuturesUnordered::new(); + fs.push(Either::Left(self.req_rx.recv_async())); + fs.push(Either::Right(self.notify_rx.notified().map(|_| { + let o: Result = Ok(AdapterRequest::Wake); + o + }))); + + match fs.next().await { + Some(Ok(a)) => a, + _ => break, + } + } else { + match self.req_rx.try_recv() { + Ok(a) => a, + Err(TryRecvError::Empty) => continue, + _ => break, + } + }; + + match msg { + AdapterRequest::Wake => blocked = false, + AdapterRequest::ByteLen => { + drop( + self.resp_tx + .send_async(AdapterResponse::ByteLen(self.stream.byte_len().await)) + .await, + ); + }, + AdapterRequest::Seek(pos) => { + pause_buf_moves = true; + drop(self.resp_tx.send_async(AdapterResponse::SeekClear).await); + seek_res = Some(self.stream.seek(pos).await); + }, + AdapterRequest::SeekCleared => { + if let Some(res) = seek_res.take() { + drop( + self.resp_tx + .send_async(AdapterResponse::SeekResult(res)) + .await, + ); + } + pause_buf_moves = false; + }, + } + } + } +} + +/// An adapter for converting an async media source into a synchronous one +/// usable by symphonia. +/// +/// This adapter takes a source implementing `AsyncRead`, and allows the receive side to +/// pass along seek requests needed. This allows for passing bytes from exclusively `AsyncRead` +/// streams (e.g., hyper HTTP sessions) to Songbird. +pub struct AsyncAdapterStream { + bytes_out: Consumer, + can_seek: bool, + // Note: this is Atomic just to work around the need for + // check_messages to take &self rather than &mut. + finalised: AtomicBool, + req_tx: Sender, + resp_rx: Receiver, + notify_tx: Arc, +} + +impl AsyncAdapterStream { + /// Wrap and pull from an async file stream, with an intermediate ring-buffer of size `buf_len` + /// between the async and sync halves. + #[must_use] + pub fn new(stream: Box, buf_len: usize) -> AsyncAdapterStream { + let (bytes_in, bytes_out) = RingBuffer::new(buf_len).split(); + let (resp_tx, resp_rx) = flume::unbounded(); + let (req_tx, req_rx) = flume::unbounded(); + let can_seek = stream.is_seekable(); + let notify_rx = Arc::new(Notify::new()); + let notify_tx = notify_rx.clone(); + + let sink = AsyncAdapterSink { + bytes_in, + req_rx, + resp_tx, + stream, + notify_rx, + }; + let stream = AsyncAdapterStream { + bytes_out, + can_seek, + finalised: false.into(), + req_tx, + resp_rx, + notify_tx, + }; + + tokio::spawn(async move { + sink.launch().await; + }); + + stream + } + + fn handle_messages(&self, block: bool) -> Option { + loop { + match self.resp_rx.try_recv() { + Ok(AdapterResponse::ReadZero) => { + self.finalised.store(true, Ordering::Relaxed); + }, + Ok(a) => break Some(a), + Err(TryRecvError::Empty) if !block => break None, + Err(TryRecvError::Disconnected) => break None, + Err(TryRecvError::Empty) => {}, + } + } + } + + fn is_dropped_and_clear(&self) -> bool { + self.resp_rx.is_empty() && self.resp_rx.is_disconnected() + } + + fn check_dropped(&self) -> IoResult<()> { + if self.is_dropped_and_clear() { + Err(IoError::new( + IoErrorKind::UnexpectedEof, + "Async half was dropped.", + )) + } else { + Ok(()) + } + } +} + +impl Read for AsyncAdapterStream { + fn read(&mut self, buf: &mut [u8]) -> IoResult { + // TODO: make this run via condvar instead? + // This needs to remain blocking or spin loopy + // Mainly because this is at odds with "keep CPU low." + loop { + drop(self.handle_messages(false)); + + match self.bytes_out.read(buf) { + Ok(n) => { + self.notify_tx.notify_one(); + return Ok(n); + }, + Err(e) if e.kind() == IoErrorKind::WouldBlock => { + // receive side must ABSOLUTELY be unblocked here. + self.notify_tx.notify_one(); + if self.finalised.load(Ordering::Relaxed) { + return Ok(0); + } + + self.check_dropped()?; + std::thread::yield_now(); + }, + a => { + println!("Misc err {:?}", a); + return a; + }, + } + } + } +} + +impl Seek for AsyncAdapterStream { + fn seek(&mut self, pos: SeekFrom) -> IoResult { + if !self.can_seek { + return Err(IoError::new( + IoErrorKind::Unsupported, + "Async half does not support seek operations.", + )); + } + + self.check_dropped()?; + + let _ = self.req_tx.send(AdapterRequest::Seek(pos)); + + // wait for async to tell us that it has stopped writing, + // then clear buf and allow async to write again. + self.finalised.store(false, Ordering::Relaxed); + match self.handle_messages(true) { + Some(AdapterResponse::SeekClear) => {}, + None => self.check_dropped().map(|_| unreachable!())?, + _ => unreachable!(), + } + + self.bytes_out.discard(self.bytes_out.capacity()); + + let _ = self.req_tx.send(AdapterRequest::SeekCleared); + + match self.handle_messages(true) { + Some(AdapterResponse::SeekResult(a)) => a, + None => self.check_dropped().map(|_| unreachable!()), + _ => unreachable!(), + } + } +} + +impl MediaSource for AsyncAdapterStream { + fn is_seekable(&self) -> bool { + self.can_seek + } + + fn byte_len(&self) -> Option { + self.check_dropped().ok()?; + + let _ = self.req_tx.send(AdapterRequest::ByteLen); + + match self.handle_messages(true) { + Some(AdapterResponse::ByteLen(a)) => a, + None => self.check_dropped().ok().map(|_| unreachable!()), + _ => unreachable!(), + } + } +} + +enum AdapterRequest { + Wake, + Seek(SeekFrom), + SeekCleared, + ByteLen, +} + +enum AdapterResponse { + SeekResult(IoResult), + SeekClear, + ByteLen(Option), + ReadZero, +} + +/// An async port of symphonia's [`MediaSource`]. +/// +/// Streams which are not seekable should implement `AsyncSeek` such that all operations +/// fail with `Unsupported`, and implement `fn is_seekable(&self) -> { false }`. +/// +/// [`MediaSource`]: MediaSource +#[async_trait] +pub trait AsyncMediaSource: AsyncRead + AsyncSeek + Send + Sync + Unpin { + /// Returns if the source is seekable. This may be an expensive operation. + fn is_seekable(&self) -> bool; + + /// Returns the length in bytes, if available. This may be an expensive operation. + async fn byte_len(&self) -> Option; + + /// Tries to recreate this stream in event of an error, resuming from the given offset. + async fn try_resume( + &mut self, + _offset: u64, + ) -> Result, AudioStreamError> { + Err(AudioStreamError::Unsupported) + } +} diff --git a/src/input/adapters/cached/compressed.rs b/src/input/adapters/cached/compressed.rs new file mode 100644 index 0000000..c15073c --- /dev/null +++ b/src/input/adapters/cached/compressed.rs @@ -0,0 +1,537 @@ +use super::{compressed_cost_per_sec, default_config, CodecCacheError, ToAudioBytes}; +use crate::{ + constants::*, + input::{ + codecs::{dca::*, CODEC_REGISTRY, PROBE}, + AudioStream, + Input, + LiveInput, + }, +}; +use audiopus::{ + coder::{Encoder as OpusEncoder, GenericCtl}, + Application, + Bitrate, + Channels, + Error as OpusError, + ErrorCode as OpusErrorCode, + SampleRate, +}; +use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; +use std::{ + convert::TryInto, + io::{ + Cursor, + Error as IoError, + ErrorKind as IoErrorKind, + Read, + Result as IoResult, + Seek, + SeekFrom, + }, + mem, + sync::atomic::{AtomicUsize, Ordering}, +}; +use streamcatcher::{ + Config as ScConfig, + NeedsBytes, + Stateful, + Transform, + TransformPosition, + TxCatcher, +}; +use symphonia_core::{ + audio::Channels as SChannels, + codecs::CodecRegistry, + io::MediaSource, + meta::{MetadataRevision, StandardTagKey, Value}, + probe::{Probe, ProbedMetadata}, +}; +use tracing::{debug, trace}; + +pub struct Config { + /// Registry of audio codecs supported by the driver. + /// + /// Defaults to [`CODEC_REGISTRY`], which adds audiopus-based Opus codec support + /// to all of Symphonia's default codecs. + /// + /// [`CODEC_REGISTRY`]: static@CODEC_REGISTRY + pub codec_registry: &'static CodecRegistry, + /// Registry of the muxers and container formats supported by the driver. + /// + /// Defaults to [`PROBE`], which includes all of Symphonia's default format handlers + /// and DCA format support. + /// + /// [`PROBE`]: static@PROBE + pub format_registry: &'static Probe, + /// Configuration for the inner streamcatcher instance. + /// + /// Notably, this governs size hints and resize logic. + pub streamcatcher: ScConfig, +} + +impl Default for Config { + fn default() -> Self { + Self { + codec_registry: &CODEC_REGISTRY, + format_registry: &PROBE, + streamcatcher: ScConfig::default(), + } + } +} + +impl Config { + pub fn default_from_cost(cost_per_sec: usize) -> Self { + let streamcatcher = default_config(cost_per_sec); + Self { + streamcatcher, + ..Default::default() + } + } +} + +/// A wrapper around an existing [`Input`] which compresses +/// the input using the Opus codec before storing it in memory. +/// +/// The main purpose of this wrapper is to enable seeking on +/// incompatible sources and to ease resource consumption for +/// commonly reused/shared tracks. If only one Opus-compressed track +/// is playing at a time, then this removes the runtime decode cost +/// from the driver. +/// +/// This is intended for use with larger, repeatedly used audio +/// tracks shared between sources, and stores the sound data +/// retrieved as **compressed Opus audio**. +/// +/// Internally, this stores the stream and its metadata as a DCA1 file, +/// which can be written out to disk for later use. +/// +/// [`Input`]: crate::input::Input +#[derive(Clone)] +pub struct Compressed { + /// Inner shared bytestore. + pub raw: TxCatcher, +} + +impl Compressed { + /// Wrap an existing [`Input`] with an in-memory store, compressed using Opus. + /// + /// [`Input`]: Input + pub async fn new(source: Input, bitrate: Bitrate) -> Result { + Self::with_config(source, bitrate, None).await + } + + /// Wrap an existing [`Input`] with an in-memory store, compressed using Opus, with + /// custom configuration for both Symphonia and the backing store. + /// + /// [`Input`]: Input + pub async fn with_config( + source: Input, + bitrate: Bitrate, + config: Option, + ) -> Result { + let input = match source { + Input::Lazy(mut r) => { + let created = if r.should_create_async() { + r.create_async().await.map_err(CodecCacheError::from) + } else { + tokio::task::spawn_blocking(move || r.create().map_err(CodecCacheError::from)) + .await + .map_err(CodecCacheError::from) + .and_then(|v| v) + }; + + created.map(LiveInput::Raw) + }, + Input::Live(LiveInput::Parsed(_), _) => Err(CodecCacheError::StreamNotAtStart), + Input::Live(a, _rec) => Ok(a), + }?; + + let cost_per_sec = compressed_cost_per_sec(bitrate); + let config = config.unwrap_or_else(|| Config::default_from_cost(cost_per_sec)); + + let promoted = tokio::task::spawn_blocking(move || { + input.promote(config.codec_registry, config.format_registry) + }) + .await??; + + // If success, guaranteed to be Parsed + let mut parsed = if let LiveInput::Parsed(parsed) = promoted { + parsed + } else { + unreachable!() + }; + + // TODO: apply length hint. + // if config.length_hint.is_none() { + // if let Some(dur) = metadata.duration { + // apply_length_hint(&mut config, dur, cost_per_sec); + // } + // } + + let track_info = parsed.decoder.codec_params(); + let chan_count = track_info.channels.map_or(2, SChannels::count); + + let (channels, stereo) = if chan_count >= 2 { + (Channels::Stereo, true) + } else { + (Channels::Mono, false) + }; + + let mut encoder = OpusEncoder::new(SampleRate::Hz48000, channels, Application::Audio)?; + encoder.set_bitrate(bitrate)?; + + let codec_type = parsed.decoder.codec_params().codec; + let encoding = config + .codec_registry + .get_codec(codec_type) + .map(|v| v.short_name.to_string()); + + let format_meta_hold = parsed.format.metadata(); + let format_meta = format_meta_hold.current(); + + let metadata = create_metadata( + &mut parsed.meta, + format_meta, + &encoder, + chan_count as u8, + encoding, + )?; + let mut metabytes = b"DCA1\0\0\0\0".to_vec(); + let orig_len = metabytes.len(); + serde_json::to_writer(&mut metabytes, &metadata)?; + let meta_len = (metabytes.len() - orig_len) + .try_into() + .map_err(|_| CodecCacheError::MetadataTooLarge)?; + + (&mut metabytes[4..][..mem::size_of::()]) + .write_i32::(meta_len) + .expect("Magic byte writing location guaranteed to be well-founded."); + + let source = ToAudioBytes::new(parsed, Some(2)); + + let raw = config + .streamcatcher + .build_tx(source, OpusCompressor::new(encoder, stereo, metabytes))?; + + Ok(Self { raw }) + } + + /// Acquire a new handle to this object, creating a new + /// view of the existing cached data from the beginning. + #[must_use] + pub fn new_handle(&self) -> Self { + Self { + raw: self.raw.new_handle(), + } + } +} + +fn create_metadata( + probe_metadata: &mut ProbedMetadata, + track_metadata: Option<&MetadataRevision>, + opus: &OpusEncoder, + channels: u8, + encoding: Option, +) -> Result { + let dca = DcaInfo { + version: 1, + tool: Tool { + name: env!("CARGO_PKG_NAME").into(), + version: env!("CARGO_PKG_VERSION").into(), + url: Some(env!("CARGO_PKG_HOMEPAGE").into()), + author: Some(env!("CARGO_PKG_AUTHORS").into()), + }, + }; + + let abr = match opus.bitrate()? { + Bitrate::BitsPerSecond(i) => Some(i as u64), + Bitrate::Auto => None, + Bitrate::Max => Some(510_000), + }; + + let mode = match opus.application()? { + Application::Voip => "voip", + Application::Audio => "music", + Application::LowDelay => "lowdelay", + } + .to_string(); + + let sample_rate = opus.sample_rate()? as u32; + + let opus = Opus { + mode, + sample_rate, + frame_size: MONO_FRAME_BYTE_SIZE as u64, + abr, + vbr: opus.vbr()?, + channels: channels.min(2), + }; + + let mut origin = Origin { + source: Some("file".into()), + abr: None, + channels: Some(channels), + encoding, + url: None, + }; + + let mut info = Info { + title: None, + artist: None, + album: None, + genre: None, + cover: None, + comments: None, + }; + + if let Some(meta) = probe_metadata.get() { + apply_meta_to_dca(&mut info, &mut origin, meta.current()); + } + + apply_meta_to_dca(&mut info, &mut origin, track_metadata); + + Ok(DcaMetadata { + dca, + opus, + info: Some(info), + origin: Some(origin), + extra: None, + }) +} + +fn apply_meta_to_dca(info: &mut Info, origin: &mut Origin, src_meta: Option<&MetadataRevision>) { + if let Some(meta) = src_meta { + for tag in meta.tags() { + match tag.std_key { + Some(StandardTagKey::Album) => + if let Value::String(s) = &tag.value { + info.album = Some(s.clone()); + }, + Some(StandardTagKey::Artist) => + if let Value::String(s) = &tag.value { + info.artist = Some(s.clone()); + }, + Some(StandardTagKey::Comment) => + if let Value::String(s) = &tag.value { + info.comments = Some(s.clone()); + }, + Some(StandardTagKey::Genre) => + if let Value::String(s) = &tag.value { + info.genre = Some(s.clone()); + }, + Some(StandardTagKey::TrackTitle) => + if let Value::String(s) = &tag.value { + info.title = Some(s.clone()); + }, + Some(StandardTagKey::Url | StandardTagKey::UrlSource) => { + if let Value::String(s) = &tag.value { + origin.url = Some(s.clone()); + } + }, + _ => {}, + } + } + + for _visual in meta.visuals() { + // FIXME: will require MIME type inspection and Base64 conversion. + } + } +} + +/// Transform applied inside [`Compressed`], converting a floating-point PCM +/// input stream into a DCA-framed Opus stream. +/// +/// Created and managed by [`Compressed`]. +/// +/// [`Compressed`]: Compressed +#[derive(Debug)] +pub struct OpusCompressor { + prepend: Option>>, + encoder: OpusEncoder, + last_frame: Vec, + stereo_input: bool, + frame_pos: usize, + audio_bytes: AtomicUsize, +} + +impl OpusCompressor { + fn new(encoder: OpusEncoder, stereo_input: bool, prepend: Vec) -> Self { + Self { + prepend: Some(Cursor::new(prepend)), + encoder, + last_frame: Vec::with_capacity(4000), + stereo_input, + frame_pos: 0, + audio_bytes: AtomicUsize::default(), + } + } +} + +impl Transform for OpusCompressor +where + T: Read, +{ + fn transform_read(&mut self, src: &mut T, buf: &mut [u8]) -> IoResult { + if let Some(prepend) = self.prepend.as_mut() { + match prepend.read(buf)? { + 0 => {}, + n => return Ok(TransformPosition::Read(n)), + } + } + + self.prepend = None; + + let output_start = mem::size_of::(); + let mut eof = false; + + let mut raw_len = 0; + let mut out = None; + let mut sample_buf = [0f32; STEREO_FRAME_SIZE]; + let (samples_in_frame, interleaved_count) = if self.stereo_input { + (STEREO_FRAME_SIZE, 2) + } else { + (MONO_FRAME_SIZE, 1) + }; + + // Purge old frame and read new, if needed. + if self.frame_pos == self.last_frame.len() + output_start || self.last_frame.is_empty() { + self.last_frame.resize(self.last_frame.capacity(), 0); + + // We can't use `read_f32_into` because we can't guarantee the buffer will be filled. + // However, we can guarantee that reads will be channel aligned at least! + for el in sample_buf[..samples_in_frame].chunks_mut(interleaved_count) { + match src.read_f32_into::(el) { + Ok(_) => { + raw_len += interleaved_count; + }, + Err(e) if e.kind() == IoErrorKind::UnexpectedEof => { + eof = true; + break; + }, + Err(e) => { + out = Some(Err(e)); + break; + }, + } + } + + if out.is_none() && raw_len > 0 { + loop { + // NOTE: we don't index by raw_len because the last frame can be too small + // to occupy a "whole packet". Zero-padding is the correct behaviour. + match self + .encoder + .encode_float(&sample_buf[..samples_in_frame], &mut self.last_frame[..]) + { + Ok(pkt_len) => { + trace!("Next packet to write has {:?}", pkt_len); + self.frame_pos = 0; + self.last_frame.truncate(pkt_len); + break; + }, + Err(OpusError::Opus(OpusErrorCode::BufferTooSmall)) => { + // If we need more capacity to encode this frame, then take it. + trace!("Resizing inner buffer (+256)."); + self.last_frame.resize(self.last_frame.len() + 256, 0); + }, + Err(e) => { + debug!("Read error {:?} {:?} {:?}.", e, out, raw_len); + out = Some(Err(IoError::new(IoErrorKind::Other, e))); + break; + }, + } + } + } + } + + if out.is_none() { + // Write from frame we have. + let start = if self.frame_pos < output_start { + (&mut buf[..output_start]) + .write_i16::(self.last_frame.len() as i16) + .expect( + "Minimum bytes requirement for Opus (2) should mean that an i16 \ + may always be written.", + ); + self.frame_pos += output_start; + + trace!("Wrote frame header: {}.", self.last_frame.len()); + + output_start + } else { + 0 + }; + + let out_pos = self.frame_pos - output_start; + let remaining = self.last_frame.len() - out_pos; + let write_len = remaining.min(buf.len() - start); + buf[start..start + write_len] + .copy_from_slice(&self.last_frame[out_pos..out_pos + write_len]); + self.frame_pos += write_len; + trace!("Appended {} to inner store", write_len); + out = Some(Ok(write_len + start)); + } + + // NOTE: use of raw_len here preserves true sample length even if + // stream is extended to 20ms boundary. + out.unwrap_or_else(|| Err(IoError::new(IoErrorKind::Other, "Unclear."))) + .map(|compressed_sz| { + self.audio_bytes + .fetch_add(raw_len * mem::size_of::(), Ordering::Release); + + if eof { + TransformPosition::Finished + } else { + TransformPosition::Read(compressed_sz) + } + }) + } +} + +impl NeedsBytes for OpusCompressor { + fn min_bytes_required(&self) -> usize { + 2 + } +} + +impl Stateful for OpusCompressor { + type State = usize; + + fn state(&self) -> Self::State { + self.audio_bytes.load(Ordering::Acquire) + } +} + +impl Read for Compressed { + fn read(&mut self, buf: &mut [u8]) -> IoResult { + self.raw.read(buf) + } +} + +impl Seek for Compressed { + fn seek(&mut self, pos: SeekFrom) -> IoResult { + self.raw.seek(pos) + } +} + +impl MediaSource for Compressed { + fn is_seekable(&self) -> bool { + true + } + + fn byte_len(&self) -> Option { + if self.raw.is_finished() { + Some(self.raw.len() as u64) + } else { + None + } + } +} + +impl From for Input { + fn from(val: Compressed) -> Input { + let input = Box::new(val); + Input::Live(LiveInput::Raw(AudioStream { input, hint: None }), None) + } +} diff --git a/src/input/adapters/cached/decompressed.rs b/src/input/adapters/cached/decompressed.rs new file mode 100644 index 0000000..5a9c14e --- /dev/null +++ b/src/input/adapters/cached/decompressed.rs @@ -0,0 +1,142 @@ +use super::{compressed::Config, CodecCacheError, ToAudioBytes}; +use crate::{ + constants::SAMPLE_RATE_RAW, + input::{AudioStream, Input, LiveInput, RawAdapter}, +}; +use std::io::{Read, Result as IoResult, Seek, SeekFrom}; +use streamcatcher::Catcher; +use symphonia_core::{audio::Channels, io::MediaSource}; + +/// A wrapper around an existing [`Input`] which caches +/// the decoded and converted audio data locally in memory +/// as `f32`-format PCM data. +/// +/// The main purpose of this wrapper is to enable seeking on +/// incompatible sources (i.e., ffmpeg output) and to ease resource +/// consumption for commonly reused/shared tracks. [`Compressed`] +/// offers similar functionality with different +/// tradeoffs. +/// +/// This is intended for use with small, repeatedly used audio +/// tracks shared between sources, and stores the sound data +/// retrieved in **uncompressed floating point** form to minimise the +/// cost of audio processing when mixing several tracks together. +/// This must be used sparingly: these cost a significant +/// *3 Mbps (375 kiB/s)*, or 131 MiB of RAM for a 6 minute song. +/// +/// [`Input`]: crate::input::Input +/// [`Compressed`]: super::Compressed +#[derive(Clone)] +pub struct Decompressed { + /// Inner shared bytestore. + pub raw: Catcher>, +} + +impl Decompressed { + /// Wrap an existing [`Input`] with an in-memory store, decompressed into `f32` PCM audio. + /// + /// [`Input`]: Input + pub async fn new(source: Input) -> Result { + Self::with_config(source, None).await + } + + /// Wrap an existing [`Input`] with an in-memory store, decompressed into `f32` PCM audio, + /// with custom configuration for both Symphonia and the backing store. + /// + /// [`Input`]: Input + pub async fn with_config( + source: Input, + config: Option, + ) -> Result { + let input = match source { + Input::Lazy(mut r) => { + let created = if r.should_create_async() { + r.create_async().await.map_err(CodecCacheError::from) + } else { + tokio::task::spawn_blocking(move || r.create().map_err(CodecCacheError::from)) + .await + .map_err(CodecCacheError::from) + .and_then(|v| v) + }; + + created.map(LiveInput::Raw) + }, + Input::Live(LiveInput::Parsed(_), _) => Err(CodecCacheError::StreamNotAtStart), + Input::Live(a, _rec) => Ok(a), + }?; + + let cost_per_sec = super::raw_cost_per_sec(true); + let config = config.unwrap_or_else(|| Config::default_from_cost(cost_per_sec)); + + let promoted = tokio::task::spawn_blocking(move || { + input.promote(config.codec_registry, config.format_registry) + }) + .await??; + + // If success, guaranteed to be Parsed + let parsed = if let LiveInput::Parsed(parsed) = promoted { + parsed + } else { + unreachable!() + }; + + let track_info = parsed.decoder.codec_params(); + let chan_count = track_info + .channels + .map(Channels::count) + .ok_or(CodecCacheError::UnknownChannelCount)?; + let sample_rate = SAMPLE_RATE_RAW as u32; + + let source = RawAdapter::new( + ToAudioBytes::new(parsed, Some(chan_count)), + sample_rate, + chan_count as u32, + ); + + let raw = config.streamcatcher.build(source)?; + + Ok(Self { raw }) + } + + /// Acquire a new handle to this object, creating a new + /// view of the existing cached data from the beginning. + #[must_use] + pub fn new_handle(&self) -> Self { + Self { + raw: self.raw.new_handle(), + } + } +} + +impl Read for Decompressed { + fn read(&mut self, buf: &mut [u8]) -> IoResult { + self.raw.read(buf) + } +} + +impl Seek for Decompressed { + fn seek(&mut self, pos: SeekFrom) -> IoResult { + self.raw.seek(pos) + } +} + +impl MediaSource for Decompressed { + fn is_seekable(&self) -> bool { + true + } + + fn byte_len(&self) -> Option { + if self.raw.is_finished() { + Some(self.raw.len() as u64) + } else { + None + } + } +} + +impl From for Input { + fn from(val: Decompressed) -> Input { + let input = Box::new(val); + Input::Live(LiveInput::Raw(AudioStream { input, hint: None }), None) + } +} diff --git a/src/input/adapters/cached/error.rs b/src/input/adapters/cached/error.rs new file mode 100644 index 0000000..e5892fb --- /dev/null +++ b/src/input/adapters/cached/error.rs @@ -0,0 +1,146 @@ +use crate::input::AudioStreamError; +use audiopus::error::Error as OpusError; +use serde_json::Error as JsonError; +use std::{ + error::Error as StdError, + fmt::{Display, Formatter, Result as FmtResult}, +}; +use streamcatcher::CatcherError; +use symphonia_core::errors::Error as SymphError; +use tokio::task::JoinError; + +/// Errors encountered using a [`Memory`] cached source. +/// +/// [`Memory`]: super::Memory +#[derive(Debug)] +pub enum Error { + /// The audio stream could not be created. + Create(AudioStreamError), + /// The audio stream failed to be created due to a panic in `spawn_blocking`. + CreatePanicked, + /// Streamcatcher's configuration was illegal, and the cache could not be created. + Streamcatcher(CatcherError), + /// The input stream had already been read (i.e., `Parsed`) and so the whole stream + /// could not be used. + StreamNotAtStart, +} + +impl Display for Error { + fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult { + match self { + Self::Create(c) => f.write_fmt(format_args!("failed to create audio stream: {}", c)), + Self::CreatePanicked => f.write_str("sync thread panicked while creating stream"), + Self::Streamcatcher(s) => + f.write_fmt(format_args!("illegal streamcatcher config: {}", s)), + Self::StreamNotAtStart => + f.write_str("stream cannot have been pre-read/parsed, missing headers"), + } + } +} + +impl StdError for Error {} + +impl From for Error { + fn from(val: AudioStreamError) -> Self { + Self::Create(val) + } +} + +impl From for Error { + fn from(val: CatcherError) -> Self { + Self::Streamcatcher(val) + } +} + +impl From for Error { + fn from(_val: JoinError) -> Self { + Self::CreatePanicked + } +} + +/// Errors encountered using a [`Compressed`] or [`Decompressed`] cached source. +/// +/// [`Compressed`]: super::Compressed +/// [`Decompressed`]: super::Decompressed +#[derive(Debug)] +pub enum CodecCacheError { + /// The audio stream could not be created. + Create(AudioStreamError), + /// Symphonia failed to parse the container or decode the default stream. + Parse(SymphError), + /// The Opus encoder could not be created. + Opus(OpusError), + /// The file's metadata could not be converted to JSON. + MetadataEncoding(JsonError), + /// The input's metadata was too large after conversion to JSON to fit in a DCA file. + MetadataTooLarge, + /// The audio stream failed to be created due to a panic in `spawn_blocking`. + CreatePanicked, + /// The audio stream's channel count could not be determined. + UnknownChannelCount, + /// Streamcatcher's configuration was illegal, and the cache could not be created. + Streamcatcher(CatcherError), + /// The input stream had already been read (i.e., `Parsed`) and so the whole stream + /// could not be used. + StreamNotAtStart, +} + +impl Display for CodecCacheError { + fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult { + match self { + Self::Create(c) => f.write_fmt(format_args!("failed to create audio stream: {}", c)), + Self::Parse(p) => f.write_fmt(format_args!("failed to parse audio format: {}", p)), + Self::Opus(o) => f.write_fmt(format_args!("failed to create Opus encoder: {}", o)), + Self::MetadataEncoding(m) => f.write_fmt(format_args!( + "failed to convert track metadata to JSON: {}", + m + )), + Self::MetadataTooLarge => f.write_str("track metadata was too large, >= 32kiB"), + Self::CreatePanicked => f.write_str("sync thread panicked while creating stream"), + Self::UnknownChannelCount => + f.write_str("audio stream's channel count could not be determined"), + Self::Streamcatcher(s) => + f.write_fmt(format_args!("illegal streamcatcher config: {}", s)), + Self::StreamNotAtStart => + f.write_str("stream cannot have been pre-read/parsed, missing headers"), + } + } +} + +impl StdError for CodecCacheError {} + +impl From for CodecCacheError { + fn from(val: AudioStreamError) -> Self { + Self::Create(val) + } +} + +impl From for CodecCacheError { + fn from(val: CatcherError) -> Self { + Self::Streamcatcher(val) + } +} + +impl From for CodecCacheError { + fn from(_val: JoinError) -> Self { + Self::CreatePanicked + } +} + +impl From for CodecCacheError { + fn from(val: JsonError) -> Self { + Self::MetadataEncoding(val) + } +} + +impl From for CodecCacheError { + fn from(val: OpusError) -> Self { + Self::Opus(val) + } +} + +impl From for CodecCacheError { + fn from(val: SymphError) -> Self { + Self::Parse(val) + } +} diff --git a/src/input/cached/hint.rs b/src/input/adapters/cached/hint.rs similarity index 100% rename from src/input/cached/hint.rs rename to src/input/adapters/cached/hint.rs diff --git a/src/input/adapters/cached/memory.rs b/src/input/adapters/cached/memory.rs new file mode 100644 index 0000000..561cfeb --- /dev/null +++ b/src/input/adapters/cached/memory.rs @@ -0,0 +1,111 @@ +use super::{default_config, raw_cost_per_sec, Error}; +use crate::input::{AudioStream, Input, LiveInput}; +use std::io::{Read, Result as IoResult, Seek}; +use streamcatcher::{Catcher, Config}; +use symphonia_core::io::MediaSource; + +/// A wrapper around an existing [`Input`] which caches its data +/// in memory. +/// +/// The main purpose of this wrapper is to enable fast seeking on +/// incompatible sources (i.e., HTTP streams) and to ease resource +/// consumption for commonly reused/shared tracks. +/// +/// This consumes exactly as many bytes of memory as the input stream contains. +/// +/// [`Input`]: Input +#[derive(Clone)] +pub struct Memory { + /// Inner shared bytestore. + pub raw: Catcher>, +} + +impl Memory { + /// Wrap an existing [`Input`] with an in-memory store with the same codec and framing. + /// + /// [`Input`]: Input + pub async fn new(source: Input) -> Result { + Self::with_config(source, None).await + } + + /// Wrap an existing [`Input`] with an in-memory store with the same codec and framing. + /// + /// `length_hint` may be used to control the size of the initial chunk, preventing + /// needless allocations and copies. + /// + /// [`Input`]: Input + pub async fn with_config(source: Input, config: Option) -> Result { + let input = match source { + Input::Lazy(mut r) => { + let created = if r.should_create_async() { + r.create_async().await + } else { + tokio::task::spawn_blocking(move || r.create()).await? + }; + + created.map(|v| v.input).map_err(Error::from) + }, + Input::Live(LiveInput::Raw(a), _rec) => Ok(a.input), + Input::Live(LiveInput::Wrapped(a), _rec) => + Ok(Box::new(a.input) as Box), + Input::Live(LiveInput::Parsed(_), _) => Err(Error::StreamNotAtStart), + }?; + + let cost_per_sec = raw_cost_per_sec(true); + + let config = config.unwrap_or_else(|| default_config(cost_per_sec)); + + // TODO: apply length hint. + // if config.length_hint.is_none() { + // if let Some(dur) = metadata.duration { + // apply_length_hint(&mut config, dur, cost_per_sec); + // } + // } + + let raw = config.build(input)?; + + Ok(Self { raw }) + } + + /// Acquire a new handle to this object, creating a new + /// view of the existing cached data from the beginning. + #[must_use] + pub fn new_handle(&self) -> Self { + Self { + raw: self.raw.new_handle(), + } + } +} + +impl Read for Memory { + fn read(&mut self, buf: &mut [u8]) -> IoResult { + self.raw.read(buf) + } +} + +impl Seek for Memory { + fn seek(&mut self, pos: std::io::SeekFrom) -> IoResult { + self.raw.seek(pos) + } +} + +impl MediaSource for Memory { + fn is_seekable(&self) -> bool { + true + } + + fn byte_len(&self) -> Option { + if self.raw.is_finished() { + Some(self.raw.len() as u64) + } else { + None + } + } +} + +impl From for Input { + fn from(val: Memory) -> Input { + let input = Box::new(val); + Input::Live(LiveInput::Raw(AudioStream { input, hint: None }), None) + } +} diff --git a/src/input/cached/mod.rs b/src/input/adapters/cached/mod.rs similarity index 88% rename from src/input/cached/mod.rs rename to src/input/adapters/cached/mod.rs index 5983c81..4a341ea 100644 --- a/src/input/cached/mod.rs +++ b/src/input/adapters/cached/mod.rs @@ -2,12 +2,14 @@ //! direct Opus frame passthrough. mod compressed; +mod decompressed; +mod error; mod hint; mod memory; -#[cfg(test)] -mod tests; +mod util; -pub use self::{compressed::*, hint::*, memory::*}; +pub(crate) use self::util::*; +pub use self::{compressed::*, decompressed::*, error::*, hint::*, memory::*}; use crate::constants::*; use crate::input::utils; @@ -16,6 +18,7 @@ use std::{mem, time::Duration}; use streamcatcher::{Config, GrowthStrategy}; /// Estimates the cost, in B/s, of audio data compressed at the given bitrate. +#[must_use] pub fn compressed_cost_per_sec(bitrate: Bitrate) -> usize { let framing_cost_per_sec = AUDIO_FRAME_RATE * mem::size_of::(); @@ -29,6 +32,7 @@ pub fn compressed_cost_per_sec(bitrate: Bitrate) -> usize { } /// Calculates the cost, in B/s, of raw floating-point audio data. +#[must_use] pub fn raw_cost_per_sec(stereo: bool) -> usize { utils::timestamp_to_byte_count(Duration::from_secs(1), stereo) } @@ -39,6 +43,7 @@ pub fn raw_cost_per_sec(stereo: bool) -> usize { /// a constant chunk size of 5s worth of audio at the given bitrate estimate. /// /// [`streamcatcher`]: https://docs.rs/streamcatcher/0.1.0/streamcatcher/struct.Config.html +#[must_use] pub fn default_config(cost_per_sec: usize) -> Config { Config::new().chunk_size(GrowthStrategy::Constant(5 * cost_per_sec)) } diff --git a/src/input/adapters/cached/util.rs b/src/input/adapters/cached/util.rs new file mode 100644 index 0000000..7167546 --- /dev/null +++ b/src/input/adapters/cached/util.rs @@ -0,0 +1,458 @@ +use crate::{constants::*, driver::tasks::mixer::mix_logic, input::Parsed}; + +use byteorder::{LittleEndian, WriteBytesExt}; +use rubato::{FftFixedOut, Resampler}; +use std::{ + io::{ErrorKind as IoErrorKind, Read, Result as IoResult, Seek, Write}, + mem, + ops::Range, +}; +use symphonia_core::{ + audio::{AudioBuffer, AudioBufferRef, Layout, Signal, SignalSpec}, + conv::IntoSample, + io::MediaSource, + sample::Sample, +}; + +const SAMPLE_LEN: usize = mem::size_of::(); + +/// Adapter for Symphonia sources into an interleaved f32 bytestream. +/// +/// This will output `f32`s in LE byte order, matching the channel count +/// of the input. +pub struct ToAudioBytes { + chan_count: usize, + chan_limit: usize, + parsed: Parsed, + /// Position with parsed's last decoded frame. + inner_pos: Range, + resample: Option, + done: bool, + + interrupted_samples: Vec, + interrupted_byte_pos: Range, +} + +struct ResampleState { + /// Used to hold outputs from resampling, *ready to be used*. + resampled_data: Vec>, + /// The actual resampler. + resampler: FftFixedOut, + /// Used to hold inputs to resampler across packet boundaries. + scratch: AudioBuffer, + /// The range of floats in `resampled_data` which have not yet + /// been read. + resample_pos: Range, +} + +impl ToAudioBytes { + pub fn new(parsed: Parsed, chan_limit: Option) -> Self { + let track_info = parsed.decoder.codec_params(); + let sample_rate = track_info.sample_rate.unwrap_or(SAMPLE_RATE_RAW as u32); + let maybe_layout = track_info.channel_layout; + let maybe_chans = track_info.channels; + + let chan_count = if let Some(chans) = maybe_chans { + chans.count() + } else if let Some(layout) = maybe_layout { + match layout { + Layout::Mono => 1, + Layout::Stereo => 2, + Layout::TwoPointOne => 3, + Layout::FivePointOne => 6, + } + } else { + 2 + }; + + let chan_limit = chan_limit.unwrap_or(chan_count); + + let resample = (sample_rate != SAMPLE_RATE_RAW as u32).then(|| { + let spec = if let Some(chans) = maybe_chans { + SignalSpec::new(SAMPLE_RATE_RAW as u32, chans) + } else if let Some(layout) = maybe_layout { + SignalSpec::new_with_layout(SAMPLE_RATE_RAW as u32, layout) + } else { + SignalSpec::new_with_layout(SAMPLE_RATE_RAW as u32, Layout::Stereo) + }; + + let scratch = AudioBuffer::::new(MONO_FRAME_SIZE as u64, spec); + + // TODO: integ. error handling here. + let resampler = FftFixedOut::new( + sample_rate as usize, + SAMPLE_RATE_RAW, + RESAMPLE_OUTPUT_FRAME_SIZE, + 4, + chan_count, + ) + .expect("Failed to create resampler."); + + let resampled_data = resampler.output_buffer_allocate(); + + ResampleState { + resampled_data, + resampler, + scratch, + resample_pos: 0..0, + } + }); + + Self { + chan_count, + chan_limit, + parsed, + inner_pos: 0..0, + resample, + done: false, + + interrupted_samples: Vec::with_capacity(chan_count), + interrupted_byte_pos: 0..0, + } + } + + pub fn num_channels(&self) -> usize { + self.chan_count.min(self.chan_limit) + } + + fn is_done(&self) -> bool { + self.done + && self.inner_pos.is_empty() + && self.resample.as_ref().map_or(true, |v| { + v.scratch.frames() == 0 && v.resample_pos.is_empty() + }) + && self.interrupted_byte_pos.is_empty() + } +} + +impl Read for ToAudioBytes { + fn read(&mut self, mut buf: &mut [u8]) -> IoResult { + // NOTE: this is disturbingly similar to the mixer code, but different enough that we can't + // just reuse it freely. + let orig_sz = buf.len(); + let num_chans = self.num_channels(); + + while !buf.is_empty() && !self.is_done() { + // Work to clear interrupted channel floats. + while !buf.is_empty() && !self.interrupted_byte_pos.is_empty() { + let index_of_first_f32 = self.interrupted_byte_pos.start / SAMPLE_LEN; + let f32_inner_pos = self.interrupted_byte_pos.start % SAMPLE_LEN; + let f32_bytes_remaining = SAMPLE_LEN - f32_inner_pos; + let to_write = f32_bytes_remaining.min(buf.len()); + + let bytes = self.interrupted_samples[index_of_first_f32].to_le_bytes(); + let written = buf.write(&bytes[f32_inner_pos..][..to_write])?; + self.interrupted_byte_pos.start += written; + } + + // Clear out already produced resampled floats. + if let Some(resample) = self.resample.as_mut() { + if !buf.is_empty() && !resample.resample_pos.is_empty() { + let bytes_advanced = write_resample_buffer( + &resample.resampled_data, + buf, + &mut resample.resample_pos, + &mut self.interrupted_samples, + &mut self.interrupted_byte_pos, + num_chans, + ); + + buf = &mut buf[bytes_advanced..]; + } + + if !resample.resample_pos.is_empty() { + continue; + } + } + + // Now work with new packets. + let source_packet = if !self.inner_pos.is_empty() { + Some(self.parsed.decoder.last_decoded()) + } else if let Ok(pkt) = self.parsed.format.next_packet() { + if pkt.track_id() != self.parsed.track_id { + continue; + } + + self.parsed + .decoder + .decode(&pkt) + .map(|pkt| { + self.inner_pos = 0..pkt.frames(); + pkt + }) + .ok() + } else { + // EOF. + None + }; + + if source_packet.is_none() { + self.done = true; + + if let Some(resample) = self.resample.as_mut() { + if resample.scratch.frames() != 0 { + let data = &mut resample.resampled_data; + let resampler = &mut resample.resampler; + let in_len = resample.scratch.frames(); + let to_render = resampler.input_frames_next().saturating_sub(in_len); + + if to_render != 0 { + resample.scratch.render_reserved(Some(to_render)); + for plane in resample.scratch.planes_mut().planes() { + for val in &mut plane[in_len..] { + *val = 0.0f32; + } + } + } + + // Luckily, we make use of the WHOLE input buffer here. + resampler + .process_into_buffer(resample.scratch.planes().planes(), data, None) + .unwrap(); + + // Calculate true end position using sample rate math + let ratio = (data[0].len() as f32) / (resample.scratch.frames() as f32); + let out_samples = (ratio * (in_len as f32)).round() as usize; + + resample.scratch.clear(); + resample.resample_pos = 0..out_samples; + } + } + + // Now go back and make use of the buffer. + // We have to do this here because we can't make any guarantees about + // the read site having enough space to hold all samples etc. + continue; + } + + let source_packet = source_packet.unwrap(); + + if let Some(resample) = self.resample.as_mut() { + // Do a resample using the newest packet. + let pkt_frames = source_packet.frames(); + + if pkt_frames == 0 { + continue; + } + + let needed_in_frames = resample.resampler.input_frames_next(); + let available_frames = self.inner_pos.len(); + + let force_copy = + resample.scratch.frames() != 0 || needed_in_frames > available_frames; + + if (!force_copy) && matches!(source_packet, AudioBufferRef::F32(_)) { + // This is the only case where we can pull off a straight resample... + // I.e., skip scratch. + + // NOTE: if let needed as if-let && {bool} is nightly only. + if let AudioBufferRef::F32(s_pkt) = source_packet { + let refs: Vec<&[f32]> = s_pkt + .planes() + .planes() + .iter() + .map(|s| &s[self.inner_pos.start..][..needed_in_frames]) + .collect(); + + self.inner_pos.start += needed_in_frames; + + resample + .resampler + .process_into_buffer(&refs, &mut resample.resampled_data, None) + .unwrap(); + } else { + unreachable!() + } + } else { + // We either lack enough samples, or have the wrong data format, forcing + // a conversion/copy into scratch. + + let old_scratch_len = resample.scratch.frames(); + let missing_frames = needed_in_frames - old_scratch_len; + let frames_to_take = available_frames.min(missing_frames); + + resample.scratch.render_reserved(Some(frames_to_take)); + mix_logic::copy_into_resampler( + &source_packet, + &mut resample.scratch, + self.inner_pos.start, + old_scratch_len, + frames_to_take, + ); + + self.inner_pos.start += frames_to_take; + + if resample.scratch.frames() == needed_in_frames { + resample + .resampler + .process_into_buffer( + resample.scratch.planes().planes(), + &mut resample.resampled_data, + None, + ) + .unwrap(); + resample.scratch.clear(); + } else { + continue; + } + } + + resample.resample_pos = 0..resample.resampled_data[0].len(); + } else { + // Newest packet may be used straight away: just convert format + // to ensure it's f32. + let bytes_advanced = write_out( + &source_packet, + buf, + &mut self.inner_pos, + &mut self.interrupted_samples, + &mut self.interrupted_byte_pos, + num_chans, + ); + + buf = &mut buf[bytes_advanced..]; + } + } + Ok(orig_sz - buf.len()) + } +} + +impl Seek for ToAudioBytes { + fn seek(&mut self, _pos: std::io::SeekFrom) -> IoResult { + Err(IoErrorKind::Unsupported.into()) + } +} + +impl MediaSource for ToAudioBytes { + fn is_seekable(&self) -> bool { + false + } + + fn byte_len(&self) -> Option { + None + } +} + +#[inline] +fn write_out( + source: &AudioBufferRef, + target: &mut [u8], + source_pos: &mut Range, + spillover: &mut Vec, + spill_range: &mut Range, + num_chans: usize, +) -> usize { + match source { + AudioBufferRef::U8(v) => + write_symph_buffer(v, target, source_pos, spillover, spill_range, num_chans), + AudioBufferRef::U16(v) => + write_symph_buffer(v, target, source_pos, spillover, spill_range, num_chans), + AudioBufferRef::U24(v) => + write_symph_buffer(v, target, source_pos, spillover, spill_range, num_chans), + AudioBufferRef::U32(v) => + write_symph_buffer(v, target, source_pos, spillover, spill_range, num_chans), + AudioBufferRef::S8(v) => + write_symph_buffer(v, target, source_pos, spillover, spill_range, num_chans), + AudioBufferRef::S16(v) => + write_symph_buffer(v, target, source_pos, spillover, spill_range, num_chans), + AudioBufferRef::S24(v) => + write_symph_buffer(v, target, source_pos, spillover, spill_range, num_chans), + AudioBufferRef::S32(v) => + write_symph_buffer(v, target, source_pos, spillover, spill_range, num_chans), + AudioBufferRef::F32(v) => + write_symph_buffer(v, target, source_pos, spillover, spill_range, num_chans), + AudioBufferRef::F64(v) => + write_symph_buffer(v, target, source_pos, spillover, spill_range, num_chans), + } +} + +#[inline] +fn write_symph_buffer( + source: &AudioBuffer, + buf: &mut [u8], + source_pos: &mut Range, + spillover: &mut Vec, + spill_range: &mut Range, + num_chans: usize, +) -> usize +where + S: Sample + IntoSample, +{ + let float_space = buf.len() / SAMPLE_LEN; + let interleaved_space = float_space / num_chans; + let non_contiguous_end = (float_space % num_chans) != 0; + + let remaining = source_pos.len(); + let to_write = remaining.min(interleaved_space); + let need_spill = non_contiguous_end && to_write < remaining; + + let samples_used = to_write + if need_spill { 1 } else { 0 }; + let last_sample = source_pos.start + to_write; + + if need_spill { + spillover.clear(); + *spill_range = 0..num_chans * SAMPLE_LEN; + } + + for (i, plane) in source.planes().planes()[..num_chans].iter().enumerate() { + for (j, sample) in plane[source_pos.start..][..to_write].iter().enumerate() { + // write this into the correct slot of buf. + let addr = ((j * num_chans) + i) * SAMPLE_LEN; + (&mut buf[addr..][..SAMPLE_LEN]) + .write_f32::((*sample).into_sample()) + .expect("Address known to exist by length checks."); + } + + if need_spill { + spillover.push(plane[last_sample].into_sample()); + } + } + + source_pos.start += samples_used; + + to_write * num_chans * SAMPLE_LEN +} + +#[inline] +fn write_resample_buffer( + source: &[Vec], + buf: &mut [u8], + source_pos: &mut Range, + spillover: &mut Vec, + spill_range: &mut Range, + num_chans: usize, +) -> usize { + let float_space = buf.len() / SAMPLE_LEN; + let interleaved_space = float_space / num_chans; + let non_contiguous_end = (float_space % num_chans) != 0; + + let remaining = source_pos.len(); + let to_write = remaining.min(interleaved_space); + let need_spill = non_contiguous_end && to_write < remaining; + + let samples_used = to_write + if need_spill { 1 } else { 0 }; + let last_sample = source_pos.start + to_write; + + if need_spill { + spillover.clear(); + *spill_range = 0..num_chans * SAMPLE_LEN; + } + + for (i, plane) in source[..num_chans].iter().enumerate() { + for (j, sample) in plane[source_pos.start..][..to_write].iter().enumerate() { + // write this into the correct slot of buf. + let addr = ((j * num_chans) + i) * SAMPLE_LEN; + (&mut buf[addr..][..SAMPLE_LEN]) + .write_f32::(*sample) + .expect("Address well-formed according to bounds checks."); + } + + if need_spill { + spillover.push(plane[last_sample]); + } + } + + source_pos.start += samples_used; + + to_write * num_chans * SAMPLE_LEN +} diff --git a/src/input/child.rs b/src/input/adapters/child.rs similarity index 73% rename from src/input/child.rs rename to src/input/adapters/child.rs index 5f92e12..daa73aa 100644 --- a/src/input/child.rs +++ b/src/input/adapters/child.rs @@ -1,9 +1,10 @@ -use super::*; +use crate::input::{AudioStream, Input, LiveInput}; use std::{ - io::{BufReader, Read}, + io::{Read, Result as IoResult}, mem, process::Child, }; +use symphonia_core::io::{MediaSource, ReadOnlySource}; use tokio::runtime::Handle; use tracing::debug; @@ -15,34 +16,7 @@ use tracing::debug; /// make sure to use `From>`. Here, the *last* process in the `Vec` will be /// used as the audio byte source. #[derive(Debug)] -pub struct ChildContainer(Vec); - -impl ChildContainer { - /// Create a new [`ChildContainer`] from a child process - pub fn new(children: Vec) -> Self { - Self(children) - } -} - -/// Create a [`Reader`] from a child process -pub fn children_to_reader(children: Vec) -> Reader { - Reader::Pipe(BufReader::with_capacity( - STEREO_FRAME_SIZE * mem::size_of::() * CHILD_BUFFER_LEN, - ChildContainer(children), - )) -} - -impl From for Reader { - fn from(container: Child) -> Self { - children_to_reader::(vec![container]) - } -} - -impl From> for Reader { - fn from(container: Vec) -> Self { - children_to_reader::(container) - } -} +pub struct ChildContainer(pub Vec); impl Read for ChildContainer { fn read(&mut self, buffer: &mut [u8]) -> IoResult { @@ -53,6 +27,36 @@ impl Read for ChildContainer { } } +impl ChildContainer { + /// Create a new [`ChildContainer`] from a child process + #[must_use] + pub fn new(children: Vec) -> Self { + Self(children) + } +} + +impl From for ChildContainer { + fn from(container: Child) -> Self { + Self(vec![container]) + } +} + +impl From> for ChildContainer { + fn from(container: Vec) -> Self { + Self(container) + } +} + +impl From for Input { + fn from(val: ChildContainer) -> Self { + let audio_stream = AudioStream { + input: Box::new(ReadOnlySource::new(val)) as Box, + hint: None, + }; + Input::Live(LiveInput::Raw(audio_stream), None) + } +} + impl Drop for ChildContainer { fn drop(&mut self) { let children = mem::take(&mut self.0); diff --git a/src/input/adapters/mod.rs b/src/input/adapters/mod.rs new file mode 100644 index 0000000..1dab4cc --- /dev/null +++ b/src/input/adapters/mod.rs @@ -0,0 +1,6 @@ +mod async_adapter; +pub mod cached; +mod child; +mod raw_adapter; + +pub use self::{async_adapter::*, child::*, raw_adapter::*}; diff --git a/src/input/adapters/raw_adapter.rs b/src/input/adapters/raw_adapter.rs new file mode 100644 index 0000000..54f0617 --- /dev/null +++ b/src/input/adapters/raw_adapter.rs @@ -0,0 +1,114 @@ +use crate::input::{AudioStream, Input, LiveInput}; +use byteorder::{LittleEndian, WriteBytesExt}; +use std::io::{ErrorKind as IoErrorKind, Read, Result as IoResult, Seek, SeekFrom, Write}; +use symphonia::core::io::MediaSource; + +// format header is a magic string, followed by two LE u32s (sample rate, channel count) +const FMT_HEADER: &[u8; 16] = b"SbirdRaw\0\0\0\0\0\0\0\0"; + +/// Adapter around a raw, interleaved, `f32` PCM byte stream. +/// +/// This may be used to port legacy songbird audio sources to be compatible with +/// the symphonia backend, particularly those with unknown length (making WAV +/// unsuitable). +/// +/// The format is described in [`RawReader`]. +/// +/// [`RawReader`]: crate::input::codecs::RawReader +pub struct RawAdapter { + prepend: [u8; 16], + inner: A, + pos: u64, +} + +impl RawAdapter { + /// Wrap an input PCM byte source to be readable by symphonia. + pub fn new(audio_source: A, sample_rate: u32, channel_count: u32) -> Self { + let mut prepend: [u8; 16] = *FMT_HEADER; + let mut write_space = &mut prepend[8..]; + + write_space + .write_u32::(sample_rate) + .expect("Prepend buffer is sized to include enough space for sample rate."); + write_space + .write_u32::(channel_count) + .expect("Prepend buffer is sized to include enough space for number of channels."); + + Self { + prepend, + inner: audio_source, + pos: 0, + } + } +} + +impl Read for RawAdapter { + fn read(&mut self, mut buf: &mut [u8]) -> IoResult { + let out = if self.pos < self.prepend.len() as u64 { + let upos = self.pos as usize; + let remaining = self.prepend.len() - upos; + let to_write = buf.len().min(remaining); + + buf.write(&self.prepend[upos..][..to_write]) + } else { + self.inner.read(buf) + }; + + if let Ok(n) = out { + self.pos += n as u64; + } + + out + } +} + +impl Seek for RawAdapter { + fn seek(&mut self, pos: SeekFrom) -> IoResult { + if self.is_seekable() { + let target_pos = match pos { + SeekFrom::Start(p) => p, + SeekFrom::End(_) => return Err(IoErrorKind::Unsupported.into()), + SeekFrom::Current(p) if p.unsigned_abs() > self.pos => + return Err(IoErrorKind::InvalidInput.into()), + SeekFrom::Current(p) => (self.pos as i64 + p) as u64, + }; + + let out = if target_pos as usize <= self.prepend.len() { + self.inner.rewind().map(|_| 0) + } else { + self.inner.seek(SeekFrom::Start(target_pos)) + }; + + match out { + Ok(0) => self.pos = target_pos, + Ok(a) => self.pos = a + self.prepend.len() as u64, + _ => {}, + } + + out.map(|_| self.pos) + } else { + Err(IoErrorKind::Unsupported.into()) + } + } +} + +impl MediaSource for RawAdapter { + fn is_seekable(&self) -> bool { + self.inner.is_seekable() + } + + fn byte_len(&self) -> Option { + self.inner.byte_len().map(|m| m + self.prepend.len() as u64) + } +} + +impl From> for Input { + fn from(val: RawAdapter) -> Self { + let live = LiveInput::Raw(AudioStream { + input: Box::new(val), + hint: None, + }); + + Input::Live(live, None) + } +} diff --git a/src/input/audiostream.rs b/src/input/audiostream.rs new file mode 100644 index 0000000..db6ff5e --- /dev/null +++ b/src/input/audiostream.rs @@ -0,0 +1,12 @@ +use symphonia_core::probe::Hint; + +/// An unread byte stream for an audio file. +pub struct AudioStream { + /// The wrapped file stream. + /// + /// An input stream *must not* have been read into past the start of the + /// audio container's header. + pub input: T, + /// Extension and MIME type information which may help guide format selection. + pub hint: Option, +} diff --git a/src/input/cached/compressed.rs b/src/input/cached/compressed.rs deleted file mode 100644 index 13c5af8..0000000 --- a/src/input/cached/compressed.rs +++ /dev/null @@ -1,303 +0,0 @@ -use super::{apply_length_hint, compressed_cost_per_sec, default_config}; -use crate::{ - constants::*, - input::{ - error::{Error, Result}, - CodecType, - Container, - Input, - Metadata, - Reader, - }, -}; -use audiopus::{ - coder::Encoder as OpusEncoder, - Application, - Bitrate, - Channels, - Error as OpusError, - ErrorCode as OpusErrorCode, - SampleRate, -}; -use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; -use std::{ - convert::TryInto, - io::{Error as IoError, ErrorKind as IoErrorKind, Read, Result as IoResult}, - mem, - sync::atomic::{AtomicUsize, Ordering}, -}; -use streamcatcher::{Config, NeedsBytes, Stateful, Transform, TransformPosition, TxCatcher}; -use tracing::{debug, trace}; - -/// A wrapper around an existing [`Input`] which compresses -/// the input using the Opus codec before storing it in memory. -/// -/// The main purpose of this wrapper is to enable seeking on -/// incompatible sources (i.e., ffmpeg output) and to ease resource -/// consumption for commonly reused/shared tracks. [`Restartable`] -/// and [`Memory`] offer the same functionality with different -/// tradeoffs. -/// -/// This is intended for use with larger, repeatedly used audio -/// tracks shared between sources, and stores the sound data -/// retrieved as **compressed Opus audio**. There is an associated memory cost, -/// but this is far smaller than using a [`Memory`]. -/// -/// [`Input`]: Input -/// [`Memory`]: super::Memory -/// [`Restartable`]: crate::input::restartable::Restartable -#[derive(Clone, Debug)] -pub struct Compressed { - /// Inner shared bytestore. - pub raw: TxCatcher, OpusCompressor>, - /// Metadata moved out of the captured source. - pub metadata: Metadata, - /// Stereo-ness of the captured source. - pub stereo: bool, -} - -impl Compressed { - /// Wrap an existing [`Input`] with an in-memory store, compressed using Opus. - /// - /// [`Input`]: Input - /// [`Metadata.duration`]: ../struct.Metadata.html#structfield.duration - pub fn new(source: Input, bitrate: Bitrate) -> Result { - Self::with_config(source, bitrate, None) - } - - /// Wrap an existing [`Input`] with an in-memory store, compressed using Opus. - /// - /// `config.length_hint` may be used to control the size of the initial chunk, preventing - /// needless allocations and copies. If this is not present, the value specified in - /// `source`'s [`Metadata::duration`] will be used. - /// - /// [`Input`]: Input - /// [`Metadata::duration`]: crate::input::Metadata::duration - pub fn with_config(source: Input, bitrate: Bitrate, config: Option) -> Result { - let channels = if source.stereo { - Channels::Stereo - } else { - Channels::Mono - }; - let mut encoder = OpusEncoder::new(SampleRate::Hz48000, channels, Application::Audio)?; - - encoder.set_bitrate(bitrate)?; - - Self::with_encoder(source, encoder, config) - } - - /// Wrap an existing [`Input`] with an in-memory store, compressed using a user-defined - /// Opus encoder. - /// - /// `length_hint` functions as in [`new`]. This function's behaviour is undefined if your encoder - /// has a different sample rate than 48kHz, and if the decoder has a different channel count from the source. - /// - /// [`Input`]: Input - /// [`new`]: Compressed::new - pub fn with_encoder( - mut source: Input, - encoder: OpusEncoder, - config: Option, - ) -> Result { - let bitrate = encoder.bitrate()?; - let cost_per_sec = compressed_cost_per_sec(bitrate); - let stereo = source.stereo; - let metadata = source.metadata.take(); - - let mut config = config.unwrap_or_else(|| default_config(cost_per_sec)); - - // apply length hint. - if config.length_hint.is_none() { - if let Some(dur) = metadata.duration { - apply_length_hint(&mut config, dur, cost_per_sec); - } - } - - let raw = config - .build_tx(Box::new(source), OpusCompressor::new(encoder, stereo)) - .map_err(Error::Streamcatcher)?; - - Ok(Self { - raw, - metadata, - stereo, - }) - } - - /// Acquire a new handle to this object, creating a new - /// view of the existing cached data from the beginning. - pub fn new_handle(&self) -> Self { - Self { - raw: self.raw.new_handle(), - metadata: self.metadata.clone(), - stereo: self.stereo, - } - } -} - -impl From for Input { - fn from(src: Compressed) -> Self { - Input::new( - true, - Reader::Compressed(src.raw), - CodecType::Opus - .try_into() - .expect("Default decoder values are known to be valid."), - Container::Dca { first_frame: 0 }, - Some(src.metadata), - ) - } -} - -/// Transform applied inside [`Compressed`], converting a floating-point PCM -/// input stream into a DCA-framed Opus stream. -/// -/// Created and managed by [`Compressed`]. -/// -/// [`Compressed`]: Compressed -#[derive(Debug)] -pub struct OpusCompressor { - encoder: OpusEncoder, - last_frame: Vec, - stereo_input: bool, - frame_pos: usize, - audio_bytes: AtomicUsize, -} - -impl OpusCompressor { - fn new(encoder: OpusEncoder, stereo_input: bool) -> Self { - Self { - encoder, - last_frame: Vec::with_capacity(4000), - stereo_input, - frame_pos: 0, - audio_bytes: Default::default(), - } - } -} - -impl Transform for OpusCompressor -where - T: Read, -{ - fn transform_read(&mut self, src: &mut T, buf: &mut [u8]) -> IoResult { - let output_start = mem::size_of::(); - let mut eof = false; - - let mut raw_len = 0; - let mut out = None; - let mut sample_buf = [0f32; STEREO_FRAME_SIZE]; - let samples_in_frame = if self.stereo_input { - STEREO_FRAME_SIZE - } else { - MONO_FRAME_SIZE - }; - - // Purge old frame and read new, if needed. - if self.frame_pos == self.last_frame.len() + output_start || self.last_frame.is_empty() { - self.last_frame.resize(self.last_frame.capacity(), 0); - - // We can't use `read_f32_into` because we can't guarantee the buffer will be filled. - for el in sample_buf[..samples_in_frame].iter_mut() { - match src.read_f32::() { - Ok(sample) => { - *el = sample; - raw_len += 1; - }, - Err(e) if e.kind() == IoErrorKind::UnexpectedEof => { - eof = true; - break; - }, - Err(e) => { - out = Some(Err(e)); - break; - }, - } - } - - if out.is_none() && raw_len > 0 { - loop { - // NOTE: we don't index by raw_len because the last frame can be too small - // to occupy a "whole packet". Zero-padding is the correct behaviour. - match self - .encoder - .encode_float(&sample_buf[..samples_in_frame], &mut self.last_frame[..]) - { - Ok(pkt_len) => { - trace!("Next packet to write has {:?}", pkt_len); - self.frame_pos = 0; - self.last_frame.truncate(pkt_len); - break; - }, - Err(OpusError::Opus(OpusErrorCode::BufferTooSmall)) => { - // If we need more capacity to encode this frame, then take it. - trace!("Resizing inner buffer (+256)."); - self.last_frame.resize(self.last_frame.len() + 256, 0); - }, - Err(e) => { - debug!("Read error {:?} {:?} {:?}.", e, out, raw_len); - out = Some(Err(IoError::new(IoErrorKind::Other, e))); - break; - }, - } - } - } - } - - if out.is_none() { - // Write from frame we have. - let start = if self.frame_pos < output_start { - (&mut buf[..output_start]) - .write_i16::(self.last_frame.len() as i16) - .expect( - "Minimum bytes requirement for Opus (2) should mean that an i16 \ - may always be written.", - ); - self.frame_pos += output_start; - - trace!("Wrote frame header: {}.", self.last_frame.len()); - - output_start - } else { - 0 - }; - - let out_pos = self.frame_pos - output_start; - let remaining = self.last_frame.len() - out_pos; - let write_len = remaining.min(buf.len() - start); - buf[start..start + write_len] - .copy_from_slice(&self.last_frame[out_pos..out_pos + write_len]); - self.frame_pos += write_len; - trace!("Appended {} to inner store", write_len); - out = Some(Ok(write_len + start)); - } - - // NOTE: use of raw_len here preserves true sample length even if - // stream is extended to 20ms boundary. - out.unwrap_or_else(|| Err(IoError::new(IoErrorKind::Other, "Unclear."))) - .map(|compressed_sz| { - self.audio_bytes - .fetch_add(raw_len * mem::size_of::(), Ordering::Release); - - if eof { - TransformPosition::Finished - } else { - TransformPosition::Read(compressed_sz) - } - }) - } -} - -impl NeedsBytes for OpusCompressor { - fn min_bytes_required(&self) -> usize { - 2 - } -} - -impl Stateful for OpusCompressor { - type State = usize; - - fn state(&self) -> Self::State { - self.audio_bytes.load(Ordering::Acquire) - } -} diff --git a/src/input/cached/memory.rs b/src/input/cached/memory.rs deleted file mode 100644 index ec79a6f..0000000 --- a/src/input/cached/memory.rs +++ /dev/null @@ -1,116 +0,0 @@ -use super::{apply_length_hint, default_config, raw_cost_per_sec}; -use crate::input::{ - error::{Error, Result}, - CodecType, - Container, - Input, - Metadata, - Reader, -}; -use std::convert::{TryFrom, TryInto}; -use streamcatcher::{Catcher, Config}; - -/// A wrapper around an existing [`Input`] which caches -/// the decoded and converted audio data locally in memory. -/// -/// The main purpose of this wrapper is to enable seeking on -/// incompatible sources (i.e., ffmpeg output) and to ease resource -/// consumption for commonly reused/shared tracks. [`Restartable`] -/// and [`Compressed`] offer the same functionality with different -/// tradeoffs. -/// -/// This is intended for use with small, repeatedly used audio -/// tracks shared between sources, and stores the sound data -/// retrieved in **uncompressed floating point** form to minimise the -/// cost of audio processing. This is a significant *3 Mbps (375 kiB/s)*, -/// or 131 MiB of RAM for a 6 minute song. -/// -/// [`Input`]: Input -/// [`Compressed`]: super::Compressed -/// [`Restartable`]: crate::input::restartable::Restartable -#[derive(Clone, Debug)] -pub struct Memory { - /// Inner shared bytestore. - pub raw: Catcher>, - /// Metadata moved out of the captured source. - pub metadata: Metadata, - /// Codec used to read the inner bytestore. - pub kind: CodecType, - /// Stereo-ness of the captured source. - pub stereo: bool, - /// Framing mechanism for the inner bytestore. - pub container: Container, -} - -impl Memory { - /// Wrap an existing [`Input`] with an in-memory store with the same codec and framing. - /// - /// [`Input`]: Input - pub fn new(source: Input) -> Result { - Self::with_config(source, None) - } - - /// Wrap an existing [`Input`] with an in-memory store with the same codec and framing. - /// - /// `length_hint` may be used to control the size of the initial chunk, preventing - /// needless allocations and copies. If this is not present, the value specified in - /// `source`'s [`Metadata::duration`] will be used, assuming that the source is uncompressed. - /// - /// [`Input`]: Input - /// [`Metadata::duration`]: crate::input::Metadata::duration - pub fn with_config(mut source: Input, config: Option) -> Result { - let stereo = source.stereo; - let kind = (&source.kind).into(); - let container = source.container; - let metadata = source.metadata.take(); - - let cost_per_sec = raw_cost_per_sec(stereo); - - let mut config = config.unwrap_or_else(|| default_config(cost_per_sec)); - - // apply length hint. - if config.length_hint.is_none() { - if let Some(dur) = metadata.duration { - apply_length_hint(&mut config, dur, cost_per_sec); - } - } - - let raw = config - .build(Box::new(source.reader)) - .map_err(Error::Streamcatcher)?; - - Ok(Self { - raw, - metadata, - kind, - stereo, - container, - }) - } - - /// Acquire a new handle to this object, creating a new - /// view of the existing cached data from the beginning. - pub fn new_handle(&self) -> Self { - Self { - raw: self.raw.new_handle(), - metadata: self.metadata.clone(), - kind: self.kind, - stereo: self.stereo, - container: self.container, - } - } -} - -impl TryFrom for Input { - type Error = Error; - - fn try_from(src: Memory) -> Result { - Ok(Input::new( - src.stereo, - Reader::Memory(src.raw), - src.kind.try_into()?, - src.container, - Some(src.metadata), - )) - } -} diff --git a/src/input/cached/tests.rs b/src/input/cached/tests.rs deleted file mode 100644 index 7e69a86..0000000 --- a/src/input/cached/tests.rs +++ /dev/null @@ -1,90 +0,0 @@ -use super::*; -use crate::{ - constants::*, - input::{error::Error, Codec, Container, Input}, - test_utils::*, -}; -use audiopus::{coder::Decoder, Bitrate, Channels, SampleRate}; -use byteorder::{LittleEndian, ReadBytesExt}; -use std::{ - convert::TryInto, - io::{Cursor, Read}, -}; - -#[tokio::test] -async fn streamcatcher_preserves_file() { - let input = make_sine(50 * MONO_FRAME_SIZE, true); - let input_len = input.len(); - - let mut raw = default_config(raw_cost_per_sec(true)) - .build(Cursor::new(input.clone())) - .map_err(Error::Streamcatcher) - .unwrap(); - - let mut out_buf = vec![]; - let read = raw.read_to_end(&mut out_buf).unwrap(); - - assert_eq!(input_len, read); - - assert_eq!(input, out_buf); -} - -#[test] -fn compressed_scans_frames_decodes_mono() { - let data = one_s_compressed_sine(false); - run_through_dca(data.raw); -} - -#[test] -fn compressed_scans_frames_decodes_stereo() { - let data = one_s_compressed_sine(true); - run_through_dca(data.raw); -} - -#[test] -fn compressed_triggers_valid_passthrough() { - let mut input = Input::from(one_s_compressed_sine(true)); - - assert!(input.supports_passthrough()); - - let mut opus_buf = [0u8; 10_000]; - let mut signal_buf = [0i16; 1920]; - - let opus_len = input.read_opus_frame(&mut opus_buf[..]).unwrap(); - - let mut decoder = Decoder::new(SampleRate::Hz48000, Channels::Stereo).unwrap(); - decoder - .decode( - Some((&opus_buf[..opus_len]).try_into().unwrap()), - (&mut signal_buf[..]).try_into().unwrap(), - false, - ) - .unwrap(); -} - -fn one_s_compressed_sine(stereo: bool) -> Compressed { - let data = make_sine(50 * MONO_FRAME_SIZE, stereo); - - let input = Input::new(stereo, data.into(), Codec::FloatPcm, Container::Raw, None); - - Compressed::new(input, Bitrate::BitsPerSecond(128_000)).unwrap() -} - -fn run_through_dca(mut src: impl Read) { - let mut decoder = Decoder::new(SampleRate::Hz48000, Channels::Stereo).unwrap(); - - let mut pkt_space = [0u8; 10_000]; - let mut signals = [0i16; 1920]; - - while let Ok(frame_len) = src.read_i16::() { - let pkt_len = src.read(&mut pkt_space[..frame_len as usize]).unwrap(); - - decoder - .decode( - Some((&pkt_space[..pkt_len]).try_into().unwrap()), - (&mut signals[..]).try_into().unwrap(), - false, - ) - .unwrap(); - } -} diff --git a/src/input/codec/mod.rs b/src/input/codec/mod.rs deleted file mode 100644 index e6d6a00..0000000 --- a/src/input/codec/mod.rs +++ /dev/null @@ -1,99 +0,0 @@ -//! Decoding schemes for input audio bytestreams. - -mod opus; - -pub use self::opus::OpusDecoderState; - -use super::*; -use std::{fmt::Debug, mem}; - -/// State used to decode input bytes of an [`Input`]. -/// -/// [`Input`]: Input -#[non_exhaustive] -#[derive(Clone, Debug)] -pub enum Codec { - /// The inner bytestream is encoded using the Opus codec, to be decoded - /// using the given state. - /// - /// Must be combined with a non-[`Raw`] container. - /// - /// [`Raw`]: Container::Raw - Opus(OpusDecoderState), - /// The inner bytestream is encoded using raw `i16` samples. - /// - /// Must be combined with a [`Raw`] container. - /// - /// [`Raw`]: Container::Raw - Pcm, - /// The inner bytestream is encoded using raw `f32` samples. - /// - /// Must be combined with a [`Raw`] container. - /// - /// [`Raw`]: Container::Raw - FloatPcm, -} - -impl From<&Codec> for CodecType { - fn from(f: &Codec) -> Self { - use Codec::*; - - match f { - Opus(_) => Self::Opus, - Pcm => Self::Pcm, - FloatPcm => Self::FloatPcm, - } - } -} - -/// Type of data being passed into an [`Input`]. -/// -/// [`Input`]: Input -#[non_exhaustive] -#[derive(Copy, Clone, Debug)] -pub enum CodecType { - /// The inner bytestream is encoded using the Opus codec. - /// - /// Must be combined with a non-[`Raw`] container. - /// - /// [`Raw`]: Container::Raw - Opus, - /// The inner bytestream is encoded using raw `i16` samples. - /// - /// Must be combined with a [`Raw`] container. - /// - /// [`Raw`]: Container::Raw - Pcm, - /// The inner bytestream is encoded using raw `f32` samples. - /// - /// Must be combined with a [`Raw`] container. - /// - /// [`Raw`]: Container::Raw - FloatPcm, -} - -impl CodecType { - /// Returns the length of a single output sample, in bytes. - pub fn sample_len(&self) -> usize { - use CodecType::*; - - match self { - Opus | FloatPcm => mem::size_of::(), - Pcm => mem::size_of::(), - } - } -} - -impl TryFrom for Codec { - type Error = Error; - - fn try_from(f: CodecType) -> Result { - use CodecType::*; - - match f { - Opus => Ok(Codec::Opus(OpusDecoderState::new()?)), - Pcm => Ok(Codec::Pcm), - FloatPcm => Ok(Codec::FloatPcm), - } - } -} diff --git a/src/input/codec/opus.rs b/src/input/codec/opus.rs deleted file mode 100644 index e5324db..0000000 --- a/src/input/codec/opus.rs +++ /dev/null @@ -1,43 +0,0 @@ -use crate::constants::*; -use audiopus::{coder::Decoder as OpusDecoder, Channels, Error as OpusError}; -use parking_lot::Mutex; -use std::sync::Arc; - -#[derive(Clone, Debug)] -/// Inner state used to decode Opus input sources. -pub struct OpusDecoderState { - /// Inner decoder used to convert opus frames into a stream of samples. - pub decoder: Arc>, - /// Controls whether this source allows direct Opus frame passthrough. - /// Defaults to `true`. - /// - /// Enabling this flag is a promise from the programmer to the audio core - /// that the source has been encoded at 48kHz, using 20ms long frames. - /// If you cannot guarantee this, disable this flag (or else risk nasal demons) - /// and bizarre audio behaviour. - pub allow_passthrough: bool, - pub(crate) current_frame: Vec, - pub(crate) frame_pos: usize, - pub(crate) should_reset: bool, -} - -impl OpusDecoderState { - /// Creates a new decoder, having stereo output at 48kHz. - pub fn new() -> Result { - Ok(Self::from_decoder(OpusDecoder::new( - SAMPLE_RATE, - Channels::Stereo, - )?)) - } - - /// Creates a new decoder pre-configured by the user. - pub fn from_decoder(decoder: OpusDecoder) -> Self { - Self { - decoder: Arc::new(Mutex::new(decoder)), - allow_passthrough: true, - current_frame: Vec::with_capacity(STEREO_FRAME_SIZE), - frame_pos: 0, - should_reset: false, - } - } -} diff --git a/src/input/codecs/dca/metadata.rs b/src/input/codecs/dca/metadata.rs new file mode 100644 index 0000000..3ffd845 --- /dev/null +++ b/src/input/codecs/dca/metadata.rs @@ -0,0 +1,53 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Deserialize, Serialize)] +pub struct DcaMetadata { + pub dca: DcaInfo, + pub opus: Opus, + pub info: Option, + pub origin: Option, + pub extra: Option, +} + +#[derive(Debug, Deserialize, Serialize)] +pub struct DcaInfo { + pub version: u64, + pub tool: Tool, +} + +#[derive(Debug, Deserialize, Serialize)] +pub struct Tool { + pub name: String, + pub version: String, + pub url: Option, + pub author: Option, +} + +#[derive(Debug, Deserialize, Serialize)] +pub struct Opus { + pub mode: String, + pub sample_rate: u32, + pub frame_size: u64, + pub abr: Option, + pub vbr: bool, + pub channels: u8, +} + +#[derive(Debug, Deserialize, Serialize)] +pub struct Info { + pub title: Option, + pub artist: Option, + pub album: Option, + pub genre: Option, + pub cover: Option, + pub comments: Option, +} + +#[derive(Debug, Deserialize, Serialize)] +pub struct Origin { + pub source: Option, + pub abr: Option, + pub channels: Option, + pub encoding: Option, + pub url: Option, +} diff --git a/src/input/codecs/dca/mod.rs b/src/input/codecs/dca/mod.rs new file mode 100644 index 0000000..8268750 --- /dev/null +++ b/src/input/codecs/dca/mod.rs @@ -0,0 +1,343 @@ +mod metadata; +pub use self::metadata::*; + +use crate::constants::{SAMPLE_RATE, SAMPLE_RATE_RAW}; + +use std::io::{Seek, SeekFrom}; +use symphonia::core::{ + codecs::{CodecParameters, CODEC_TYPE_OPUS}, + errors::{self as symph_err, Error as SymphError, Result as SymphResult, SeekErrorKind}, + formats::prelude::*, + io::{MediaSource, MediaSourceStream, ReadBytes, SeekBuffered}, + meta::{Metadata as SymphMetadata, MetadataBuilder, MetadataLog, StandardTagKey, Tag, Value}, + probe::{Descriptor, Instantiate, QueryDescriptor}, + sample::SampleFormat, + units::TimeStamp, +}; + +impl QueryDescriptor for DcaReader { + fn query() -> &'static [Descriptor] { + &[symphonia_core::support_format!( + "dca", + "DCA[0/1] Opus Wrapper", + &["dca"], + &[], + &[b"DCA1"] + )] + } + + fn score(_context: &[u8]) -> u8 { + 255 + } +} + +struct SeekAccel { + frame_offsets: Vec<(TimeStamp, u64)>, + seek_index_fill_rate: u16, + next_ts: TimeStamp, +} + +impl SeekAccel { + fn new(options: FormatOptions, first_frame_byte_pos: u64) -> Self { + let per_s = options.seek_index_fill_rate; + let next_ts = (per_s as u64) * (SAMPLE_RATE_RAW as u64); + + Self { + frame_offsets: vec![(0, first_frame_byte_pos)], + seek_index_fill_rate: per_s, + next_ts, + } + } + + fn update(&mut self, ts: TimeStamp, pos: u64) { + if ts >= self.next_ts { + self.next_ts += (self.seek_index_fill_rate as u64) * (SAMPLE_RATE_RAW as u64); + self.frame_offsets.push((ts, pos)); + } + } + + fn get_seek_pos(&self, ts: TimeStamp) -> (TimeStamp, u64) { + let index = self.frame_offsets.partition_point(|&(o_ts, _)| o_ts <= ts) - 1; + self.frame_offsets[index] + } +} + +/// [DCA\[0/1\]](https://github.com/bwmarrin/dca) Format reader for Symphonia. +pub struct DcaReader { + source: MediaSourceStream, + track: Option, + metas: MetadataLog, + seek_accel: SeekAccel, + curr_ts: TimeStamp, + max_ts: Option, + held_packet: Option, +} + +impl FormatReader for DcaReader { + fn try_new(mut source: MediaSourceStream, options: &FormatOptions) -> SymphResult { + // Read in the magic number to verify it's a DCA file. + let magic = source.read_quad_bytes()?; + + // FIXME: make use of the new options.enable_gapless to apply the opus coder delay. + + let read_meta = match &magic { + b"DCA1" => true, + _ if &magic[..3] == b"DCA" => { + return symph_err::unsupported_error("unsupported DCA version"); + }, + _ => { + source.seek_buffered_rel(-4); + false + }, + }; + + let mut codec_params = CodecParameters::new(); + + codec_params + .for_codec(CODEC_TYPE_OPUS) + .with_max_frames_per_packet(1) + .with_sample_rate(SAMPLE_RATE_RAW as u32) + .with_time_base(TimeBase::new(1, SAMPLE_RATE_RAW as u32)) + .with_sample_format(SampleFormat::F32); + + let mut metas = MetadataLog::default(); + + if read_meta { + let size = source.read_u32()?; + + // Sanity check + if (size as i32) < 2 { + return symph_err::decode_error("missing DCA1 metadata block"); + } + + let raw_json = source.read_boxed_slice_exact(size as usize)?; + + let metadata: DcaMetadata = serde_json::from_slice::(&raw_json) + .map_err(|_| SymphError::DecodeError("malformed DCA1 metadata block"))?; + + let mut revision = MetadataBuilder::new(); + + if let Some(info) = metadata.info { + if let Some(t) = info.title { + revision.add_tag(Tag::new( + Some(StandardTagKey::TrackTitle), + "title", + Value::String(t), + )); + } + if let Some(t) = info.album { + revision.add_tag(Tag::new( + Some(StandardTagKey::Album), + "album", + Value::String(t), + )); + } + if let Some(t) = info.artist { + revision.add_tag(Tag::new( + Some(StandardTagKey::Artist), + "artist", + Value::String(t), + )); + } + if let Some(t) = info.genre { + revision.add_tag(Tag::new( + Some(StandardTagKey::Genre), + "genre", + Value::String(t), + )); + } + if let Some(t) = info.comments { + revision.add_tag(Tag::new( + Some(StandardTagKey::Comment), + "comments", + Value::String(t), + )); + } + if let Some(_t) = info.cover { + // TODO: Add visual, figure out MIME types. + } + } + + if let Some(origin) = metadata.origin { + if let Some(t) = origin.url { + revision.add_tag(Tag::new(Some(StandardTagKey::Url), "url", Value::String(t))); + } + } + + metas.push(revision.metadata()); + } + + let bytes_read = source.pos(); + + Ok(Self { + source, + track: Some(Track { + id: 0, + language: None, + codec_params, + }), + metas, + seek_accel: SeekAccel::new(*options, bytes_read), + curr_ts: 0, + max_ts: None, + held_packet: None, + }) + } + + fn cues(&self) -> &[Cue] { + // No cues in DCA... + &[] + } + + fn metadata(&mut self) -> SymphMetadata<'_> { + self.metas.metadata() + } + + fn seek(&mut self, _mode: SeekMode, to: SeekTo) -> SymphResult { + let can_backseek = self.source.is_seekable(); + + let track = if self.track.is_none() { + return symph_err::seek_error(SeekErrorKind::Unseekable); + } else { + self.track.as_ref().unwrap() + }; + + let rate = track.codec_params.sample_rate; + let ts = match to { + SeekTo::Time { time, .. } => + if let Some(rate) = rate { + TimeBase::new(1, rate).calc_timestamp(time) + } else { + return symph_err::seek_error(SeekErrorKind::Unseekable); + }, + SeekTo::TimeStamp { ts, .. } => ts, + }; + + if let Some(max_ts) = self.max_ts { + if ts > max_ts { + return symph_err::seek_error(SeekErrorKind::OutOfRange); + } + } + + let backseek_needed = self.curr_ts > ts; + + if backseek_needed && !can_backseek { + return symph_err::seek_error(SeekErrorKind::ForwardOnly); + } + + let (accel_seek_ts, accel_seek_pos) = self.seek_accel.get_seek_pos(ts); + + if backseek_needed || accel_seek_pos > self.source.pos() { + self.source.seek(SeekFrom::Start(accel_seek_pos))?; + self.curr_ts = accel_seek_ts; + } + + while let Ok(pkt) = self.next_packet() { + let pts = pkt.ts; + let dur = pkt.dur; + let track_id = pkt.track_id(); + + if (pts..pts + dur).contains(&ts) { + self.held_packet = Some(pkt); + return Ok(SeekedTo { + track_id, + required_ts: ts, + actual_ts: pts, + }); + } + } + + symph_err::seek_error(SeekErrorKind::OutOfRange) + } + + fn tracks(&self) -> &[Track] { + // DCA tracks can hold only one track by design. + // Of course, a zero-length file is technically allowed, + // in which case no track. + if let Some(track) = self.track.as_ref() { + std::slice::from_ref(track) + } else { + &[] + } + } + + fn default_track(&self) -> Option<&Track> { + self.track.as_ref() + } + + fn next_packet(&mut self) -> SymphResult { + if let Some(pkt) = self.held_packet.take() { + return Ok(pkt); + } + + let frame_pos = self.source.pos(); + + let p_len = match self.source.read_u16() { + Ok(len) => len as i16, + Err(eof) => { + self.max_ts = Some(self.curr_ts); + return Err(eof.into()); + }, + }; + + if p_len < 0 { + return symph_err::decode_error("DCA frame header had a negative length."); + } + + let buf = self.source.read_boxed_slice_exact(p_len as usize)?; + + let checked_buf = buf[..].try_into().or_else(|_| { + symph_err::decode_error("Packet was not a valid Opus Packet: too large for audiopus.") + })?; + + let sample_ct = audiopus::packet::nb_samples(checked_buf, SAMPLE_RATE).or_else(|_| { + symph_err::decode_error( + "Packet was not a valid Opus packet: couldn't read sample count.", + ) + })? as u64; + + let out = Packet::new_from_boxed_slice(0, self.curr_ts, sample_ct, buf); + + self.seek_accel.update(self.curr_ts, frame_pos); + + self.curr_ts += sample_ct; + + Ok(out) + } + + fn into_inner(self: Box) -> MediaSourceStream { + self.source + } +} + +#[cfg(test)] +mod tests { + use crate::input::input_tests::*; + use crate::{constants::test_data::FILE_DCA_TARGET, input::File}; + + // NOTE: this covers youtube audio in a non-copyright-violating way, since + // those depend on an HttpRequest internally anyhow. + #[tokio::test] + #[ntest::timeout(10_000)] + async fn dca_track_plays() { + track_plays_passthrough(|| File::new(FILE_DCA_TARGET)).await; + } + + #[tokio::test] + #[ntest::timeout(10_000)] + async fn dca_forward_seek_correct() { + forward_seek_correct(|| File::new(FILE_DCA_TARGET)).await; + } + + #[tokio::test] + #[ntest::timeout(10_000)] + async fn dca_backward_seek_correct() { + backward_seek_correct(|| File::new(FILE_DCA_TARGET)).await; + } + + #[tokio::test] + #[ntest::timeout(10_000)] + async fn opus_passthrough_when_other_tracks_paused() { + track_plays_passthrough_when_is_only_active(|| File::new(FILE_DCA_TARGET)).await; + } +} diff --git a/src/input/codecs/mod.rs b/src/input/codecs/mod.rs new file mode 100644 index 0000000..b919816 --- /dev/null +++ b/src/input/codecs/mod.rs @@ -0,0 +1,34 @@ +//! Codec registries extending Symphonia's probe and registry formats with Opus and DCA support. + +pub(crate) mod dca; +mod opus; +mod raw; + +pub use self::{dca::DcaReader, opus::OpusDecoder, raw::*}; +use lazy_static::lazy_static; +use symphonia::{ + core::{codecs::CodecRegistry, probe::Probe}, + default::*, +}; + +lazy_static! { + /// Default Symphonia CodecRegistry, including the (audiopus-backed) + /// Opus codec. + pub static ref CODEC_REGISTRY: CodecRegistry = { + let mut registry = CodecRegistry::new(); + register_enabled_codecs(&mut registry); + registry.register_all::(); + registry + }; +} + +lazy_static! { + /// Default Symphonia Probe, including DCA format support. + pub static ref PROBE: Probe = { + let mut probe = Probe::default(); + probe.register_all::(); + probe.register_all::(); + register_enabled_formats(&mut probe); + probe + }; +} diff --git a/src/input/codecs/opus.rs b/src/input/codecs/opus.rs new file mode 100644 index 0000000..51efa65 --- /dev/null +++ b/src/input/codecs/opus.rs @@ -0,0 +1,167 @@ +use crate::constants::*; +use audiopus::{ + coder::{Decoder as AudiopusDecoder, GenericCtl}, + Channels, + Error as OpusError, + ErrorCode, +}; +use symphonia_core::{ + audio::{AsAudioBufferRef, AudioBuffer, AudioBufferRef, Layout, Signal, SignalSpec}, + codecs::{ + CodecDescriptor, + CodecParameters, + Decoder, + DecoderOptions, + FinalizeResult, + CODEC_TYPE_OPUS, + }, + errors::{decode_error, Result as SymphResult}, + formats::Packet, +}; + +/// Opus decoder for symphonia, based on libopus v1.3 (via [`audiopus`]). +pub struct OpusDecoder { + inner: AudiopusDecoder, + params: CodecParameters, + buf: AudioBuffer, + rawbuf: Vec, +} + +/// # SAFETY +/// The underlying Opus decoder (currently) requires only a `&self` parameter +/// to decode given packets, which is likely a mistaken decision. +/// +/// This struct makes stronger assumptions and only touches FFI decoder state with a +/// `&mut self`, preventing data races via `&OpusDecoder` as required by `impl Sync`. +/// No access to other internal state relies on unsafety or crosses FFI. +unsafe impl Sync for OpusDecoder {} + +impl OpusDecoder { + fn decode_inner(&mut self, packet: &Packet) -> SymphResult<()> { + let s_ct = loop { + let pkt = if packet.buf().is_empty() { + None + } else if let Ok(checked_pkt) = packet.buf().try_into() { + Some(checked_pkt) + } else { + return decode_error("Opus packet was too large (greater than i32::MAX bytes)."); + }; + let out_space = (&mut self.rawbuf[..]).try_into().expect("The following logic expands this buffer safely below i32::MAX, and we throw our own error."); + + match self.inner.decode_float(pkt, out_space, false) { + Ok(v) => break v, + Err(OpusError::Opus(ErrorCode::BufferTooSmall)) => { + // double the buffer size + // correct behav would be to mirror the decoder logic in the udp_rx set. + let new_size = (self.rawbuf.len() * 2).min(std::i32::MAX as usize); + if new_size == self.rawbuf.len() { + return decode_error("Opus frame too big: cannot expand opus frame decode buffer any further."); + } + + self.rawbuf.resize(new_size, 0.0); + self.buf = AudioBuffer::new( + self.rawbuf.len() as u64 / 2, + SignalSpec::new_with_layout(SAMPLE_RATE_RAW as u32, Layout::Stereo), + ); + }, + Err(e) => { + tracing::error!("Opus decode error: {:?}", e); + return decode_error("Opus decode error: see 'tracing' logs."); + }, + } + }; + + self.buf.clear(); + self.buf.render_reserved(Some(s_ct)); + + // Forcibly assuming stereo, for now. + for ch in 0..2 { + let iter = self.rawbuf.chunks_exact(2).map(|chunk| chunk[ch]); + for (tgt, src) in self.buf.chan_mut(ch).iter_mut().zip(iter) { + *tgt = src; + } + } + + Ok(()) + } +} + +impl Decoder for OpusDecoder { + fn try_new(params: &CodecParameters, _options: &DecoderOptions) -> SymphResult { + let inner = AudiopusDecoder::new(SAMPLE_RATE, Channels::Stereo).unwrap(); + + let mut params = params.clone(); + params.with_sample_rate(SAMPLE_RATE_RAW as u32); + + Ok(Self { + inner, + params, + buf: AudioBuffer::new( + MONO_FRAME_SIZE as u64, + SignalSpec::new_with_layout(SAMPLE_RATE_RAW as u32, Layout::Stereo), + ), + rawbuf: vec![0.0f32; STEREO_FRAME_SIZE], + }) + } + + fn supported_codecs() -> &'static [CodecDescriptor] { + &[symphonia_core::support_codec!( + CODEC_TYPE_OPUS, + "opus", + "libopus (1.3+, audiopus)" + )] + } + + fn codec_params(&self) -> &CodecParameters { + &self.params + } + + fn decode(&mut self, packet: &Packet) -> SymphResult> { + if let Err(e) = self.decode_inner(packet) { + self.buf.clear(); + Err(e) + } else { + Ok(self.buf.as_audio_buffer_ref()) + } + } + + fn reset(&mut self) { + let _ = self.inner.reset_state(); + } + + fn finalize(&mut self) -> FinalizeResult { + FinalizeResult::default() + } + + fn last_decoded(&self) -> AudioBufferRef { + self.buf.as_audio_buffer_ref() + } +} + +#[cfg(test)] +mod tests { + use crate::{ + constants::test_data::FILE_WEBM_TARGET, + input::{input_tests::*, File}, + }; + + // NOTE: this covers youtube audio in a non-copyright-violating way, since + // those depend on an HttpRequest internally anyhow. + #[tokio::test] + #[ntest::timeout(10_000)] + async fn webm_track_plays() { + track_plays_passthrough(|| File::new(FILE_WEBM_TARGET)).await; + } + + #[tokio::test] + #[ntest::timeout(10_000)] + async fn webm_forward_seek_correct() { + forward_seek_correct(|| File::new(FILE_WEBM_TARGET)).await; + } + + #[tokio::test] + #[ntest::timeout(10_000)] + async fn webm_backward_seek_correct() { + backward_seek_correct(|| File::new(FILE_WEBM_TARGET)).await; + } +} diff --git a/src/input/codecs/raw.rs b/src/input/codecs/raw.rs new file mode 100644 index 0000000..0076b3f --- /dev/null +++ b/src/input/codecs/raw.rs @@ -0,0 +1,182 @@ +use std::io::{Seek, SeekFrom}; +use symphonia::core::{ + audio::Channels, + codecs::{CodecParameters, CODEC_TYPE_PCM_F32LE}, + errors::{self as symph_err, Result as SymphResult, SeekErrorKind}, + formats::prelude::*, + io::{MediaSource, MediaSourceStream, ReadBytes, SeekBuffered}, + meta::{Metadata as SymphMetadata, MetadataLog}, + probe::{Descriptor, Instantiate, QueryDescriptor}, + units::TimeStamp, +}; + +impl QueryDescriptor for RawReader { + fn query() -> &'static [Descriptor] { + &[symphonia_core::support_format!( + "raw", + "Raw arbitrary-length f32 audio container.", + &["rawf32"], + &[], + &[b"SbirdRaw"] + )] + } + + fn score(_context: &[u8]) -> u8 { + 255 + } +} + +/// Symphonia support for a simple container for raw f32-PCM data of unknown duration. +/// +/// Contained files have a simple header: +/// * the 8-byte signature `b"SbirdRaw"`, +/// * the sample rate, as a little-endian `u32`, +/// * the channel count, as a little-endian `u32`. +/// +/// The remainder of the file is interleaved little-endian `f32` samples. +pub struct RawReader { + source: MediaSourceStream, + track: Track, + meta: MetadataLog, + curr_ts: TimeStamp, + max_ts: Option, +} + +impl FormatReader for RawReader { + fn try_new(mut source: MediaSourceStream, _options: &FormatOptions) -> SymphResult { + let mut magic = [0u8; 8]; + ReadBytes::read_buf_exact(&mut source, &mut magic[..])?; + + if &magic != b"SbirdRaw" { + source.seek_buffered_rel(-(magic.len() as isize)); + return symph_err::decode_error("rawf32: illegal magic byte sequence."); + } + + let sample_rate = source.read_u32()?; + let n_chans = source.read_u32()?; + + let chans = match n_chans { + 1 => Channels::FRONT_LEFT, + 2 => Channels::FRONT_LEFT | Channels::FRONT_RIGHT, + _ => + return symph_err::decode_error( + "rawf32: channel layout is not stereo or mono for fmt_pcm", + ), + }; + + let mut codec_params = CodecParameters::new(); + + codec_params + .for_codec(CODEC_TYPE_PCM_F32LE) + .with_bits_per_coded_sample((std::mem::size_of::() as u32) * 8) + .with_bits_per_sample((std::mem::size_of::() as u32) * 8) + .with_sample_rate(sample_rate) + .with_time_base(TimeBase::new(1, sample_rate)) + .with_sample_format(symphonia_core::sample::SampleFormat::F32) + .with_max_frames_per_packet(sample_rate as u64 / 50) + .with_channels(chans); + + Ok(Self { + source, + track: Track { + id: 0, + language: None, + codec_params, + }, + meta: MetadataLog::default(), + curr_ts: 0, + max_ts: None, + }) + } + + fn cues(&self) -> &[Cue] { + &[] + } + + fn metadata(&mut self) -> SymphMetadata<'_> { + self.meta.metadata() + } + + fn seek(&mut self, _mode: SeekMode, to: SeekTo) -> SymphResult { + let can_backseek = self.source.is_seekable(); + + let track = &self.track; + let rate = track.codec_params.sample_rate; + let ts = match to { + SeekTo::Time { time, .. } => + if let Some(rate) = rate { + TimeBase::new(1, rate).calc_timestamp(time) + } else { + return symph_err::seek_error(SeekErrorKind::Unseekable); + }, + SeekTo::TimeStamp { ts, .. } => ts, + }; + + if let Some(max_ts) = self.max_ts { + if ts > max_ts { + return symph_err::seek_error(SeekErrorKind::OutOfRange); + } + } + + let backseek_needed = self.curr_ts > ts; + + if backseek_needed && !can_backseek { + return symph_err::seek_error(SeekErrorKind::ForwardOnly); + } + + let chan_count = track + .codec_params + .channels + .expect("Channel count is built into format.") + .count() as u64; + + let seek_pos = 16 + (std::mem::size_of::() as u64) * (ts * chan_count); + + self.source.seek(SeekFrom::Start(seek_pos))?; + self.curr_ts = ts; + + Ok(SeekedTo { + track_id: track.id, + required_ts: ts, + actual_ts: ts, + }) + } + + fn tracks(&self) -> &[Track] { + std::slice::from_ref(&self.track) + } + + fn default_track(&self) -> Option<&Track> { + Some(&self.track) + } + + fn next_packet(&mut self) -> SymphResult { + let track = &self.track; + let rate = track + .codec_params + .sample_rate + .expect("Sample rate is built into format.") as usize; + + let chan_count = track + .codec_params + .channels + .expect("Channel count is built into format.") + .count(); + + let sample_unit = std::mem::size_of::() * chan_count; + + // Aim for 20ms (50Hz). + let buf = self.source.read_boxed_slice((rate / 50) * sample_unit)?; + + let sample_ct = (buf.len() / sample_unit) as u64; + let out = Packet::new_from_boxed_slice(0, self.curr_ts, sample_ct, buf); + + self.curr_ts += sample_ct; + + Ok(out) + } + + fn into_inner(self: Box) -> MediaSourceStream { + self.source + } +} diff --git a/src/input/compose.rs b/src/input/compose.rs new file mode 100644 index 0000000..0b51bba --- /dev/null +++ b/src/input/compose.rs @@ -0,0 +1,40 @@ +use super::{AudioStream, AudioStreamError, AuxMetadata}; + +use symphonia_core::io::MediaSource; + +/// Data and behaviour required to instantiate a lazy audio source. +#[async_trait::async_trait] +pub trait Compose: Send { + /// Create a source synchronously. + /// + /// If [`should_create_async`] returns `false`, this method will chosen at runtime. + /// + /// [`should_create_async`]: Self::should_create_async + fn create(&mut self) -> Result>, AudioStreamError>; + + /// Create a source asynchronously. + /// + /// If [`should_create_async`] returns `true`, this method will chosen at runtime. + /// + /// [`should_create_async`]: Self::should_create_async + async fn create_async(&mut self) + -> Result>, AudioStreamError>; + + /// Determines whether this source will be instantiated using [`create`] or [`create_async`]. + /// + /// Songbird will create the audio stream using either a dynamically sized thread pool, + /// or a task on the async runtime it was spawned in respectively. Users do not need to + /// support both these methods. + /// + /// [`create_async`]: Self::create_async + /// [`create`]: Self::create + fn should_create_async(&self) -> bool; + + /// Requests auxiliary metadata which can be accessed without parsing the file. + /// + /// This method will never be called by songbird but allows, for instance, access to metadata + /// which might only be visible to a web crawler e.g., uploader or source URL. + async fn aux_metadata(&mut self) -> Result { + Err(AudioStreamError::Unsupported) + } +} diff --git a/src/input/container/frame.rs b/src/input/container/frame.rs deleted file mode 100644 index fb5f0f4..0000000 --- a/src/input/container/frame.rs +++ /dev/null @@ -1,8 +0,0 @@ -/// Information used in audio frame detection. -#[derive(Clone, Copy, Debug)] -pub struct Frame { - /// Length of this frame's header, in bytes. - pub header_len: usize, - /// Payload length, in bytes. - pub frame_len: usize, -} diff --git a/src/input/container/mod.rs b/src/input/container/mod.rs deleted file mode 100644 index f22b013..0000000 --- a/src/input/container/mod.rs +++ /dev/null @@ -1,69 +0,0 @@ -mod frame; - -pub use frame::*; - -use super::CodecType; -use byteorder::{LittleEndian, ReadBytesExt}; -use std::{ - fmt::Debug, - io::{Read, Result as IoResult}, - mem, -}; - -/// Marker and state for decoding framed input files. -#[non_exhaustive] -#[derive(Clone, Copy, Debug)] -pub enum Container { - /// Raw, unframed input. - Raw, - /// Framed input, beginning with a JSON header. - /// - /// Frames have the form `{ len: i16, payload: [u8; len]}`. - Dca { - /// Byte index of the first frame after the JSON header. - first_frame: usize, - }, -} - -impl Container { - /// Tries to read the header of the next frame from an input stream. - pub fn next_frame_length( - &mut self, - mut reader: impl Read, - input: CodecType, - ) -> IoResult { - use Container::*; - - match self { - Raw => Ok(Frame { - header_len: 0, - frame_len: input.sample_len(), - }), - Dca { .. } => reader.read_i16::().map(|frame_len| Frame { - header_len: mem::size_of::(), - frame_len: frame_len.max(0) as usize, - }), - } - } - - /// Tries to seek on an input directly using sample length, if the input - /// is unframed. - pub fn try_seek_trivial(&self, input: CodecType) -> Option { - use Container::*; - - match self { - Raw => Some(input.sample_len()), - _ => None, - } - } - - /// Returns the byte index of the first frame containing audio payload data. - pub fn input_start(&self) -> usize { - use Container::*; - - match self { - Raw => 0, - Dca { first_frame } => *first_frame, - } - } -} diff --git a/src/input/dca.rs b/src/input/dca.rs deleted file mode 100644 index ee2e14f..0000000 --- a/src/input/dca.rs +++ /dev/null @@ -1,143 +0,0 @@ -use super::{codec::OpusDecoderState, error::DcaError, Codec, Container, Input, Metadata, Reader}; -use serde::Deserialize; -use std::{ffi::OsStr, mem}; -use tokio::{fs::File as TokioFile, io::AsyncReadExt}; - -/// Creates a streamed audio source from a DCA file. -/// Currently only accepts the [DCA1 format](https://github.com/bwmarrin/dca). -pub async fn dca>(path: P) -> Result { - _dca(path.as_ref()).await -} - -async fn _dca(path: &OsStr) -> Result { - let mut reader = TokioFile::open(path).await.map_err(DcaError::IoError)?; - - let mut header = [0u8; 4]; - - // Read in the magic number to verify it's a DCA file. - reader - .read_exact(&mut header) - .await - .map_err(DcaError::IoError)?; - - if header != b"DCA1"[..] { - return Err(DcaError::InvalidHeader); - } - - let size = reader - .read_i32_le() - .await - .map_err(|_| DcaError::InvalidHeader)?; - - // Sanity check - if size < 2 { - return Err(DcaError::InvalidSize(size)); - } - - let mut raw_json = Vec::with_capacity(size as usize); - - let mut json_reader = reader.take(size as u64); - - json_reader - .read_to_end(&mut raw_json) - .await - .map_err(DcaError::IoError)?; - - let reader = json_reader.into_inner().into_std().await; - - let metadata: Metadata = serde_json::from_slice::(raw_json.as_slice()) - .map_err(DcaError::InvalidMetadata)? - .into(); - - let stereo = metadata.channels == Some(2); - - Ok(Input::new( - stereo, - Reader::from_file(reader), - Codec::Opus(OpusDecoderState::new().map_err(DcaError::Opus)?), - Container::Dca { - first_frame: (size as usize) + mem::size_of::() + header.len(), - }, - Some(metadata), - )) -} - -#[allow(dead_code)] -#[derive(Debug, Deserialize)] -pub(crate) struct DcaMetadata { - pub(crate) dca: Dca, - pub(crate) opus: Opus, - pub(crate) info: Option, - pub(crate) origin: Option, - pub(crate) extra: Option, -} - -#[allow(dead_code)] -#[derive(Debug, Deserialize)] -pub(crate) struct Dca { - pub(crate) version: u64, - pub(crate) tool: Tool, -} - -#[allow(dead_code)] -#[derive(Debug, Deserialize)] -pub(crate) struct Tool { - pub(crate) name: String, - pub(crate) version: String, - pub(crate) url: String, - pub(crate) author: String, -} - -#[allow(dead_code)] -#[derive(Debug, Deserialize)] -pub(crate) struct Opus { - pub(crate) mode: String, - pub(crate) sample_rate: u32, - pub(crate) frame_size: u64, - pub(crate) abr: u64, - pub(crate) vbr: u64, - pub(crate) channels: u8, -} - -#[allow(dead_code)] -#[derive(Debug, Deserialize)] -pub(crate) struct Info { - pub(crate) title: Option, - pub(crate) artist: Option, - pub(crate) album: Option, - pub(crate) genre: Option, - pub(crate) cover: Option, -} - -#[allow(dead_code)] -#[derive(Debug, Deserialize)] -pub(crate) struct Origin { - pub(crate) source: Option, - pub(crate) abr: Option, - pub(crate) channels: Option, - pub(crate) encoding: Option, - pub(crate) url: Option, -} - -impl From for Metadata { - fn from(mut d: DcaMetadata) -> Self { - let (track, artist) = d - .info - .take() - .map(|mut m| (m.title.take(), m.artist.take())) - .unwrap_or_else(|| (None, None)); - - let channels = Some(d.opus.channels); - let sample_rate = Some(d.opus.sample_rate); - - Self { - track, - artist, - - channels, - sample_rate, - - ..Default::default() - } - } -} diff --git a/src/input/error.rs b/src/input/error.rs index 551ea36..7072584 100644 --- a/src/input/error.rs +++ b/src/input/error.rs @@ -1,159 +1,169 @@ -//! Errors caused by input creation. +use std::{error::Error, fmt::Display, time::Duration}; +use symphonia_core::errors::Error as SymphError; -use audiopus::Error as OpusError; -use core::fmt; -use serde_json::{Error as JsonError, Value}; -use std::{error::Error as StdError, io::Error as IoError, process::Output}; -use streamcatcher::CatcherError; - -/// An error returned when creating a new [`Input`]. +/// Errors encountered when creating an [`AudioStream`] or requesting metadata +/// from a [`Compose`]. /// -/// [`Input`]: crate::input::Input -#[derive(Debug)] +/// [`AudioStream`]: super::AudioStream +/// [`Compose`]: super::Compose #[non_exhaustive] -pub enum Error { - /// An error occurred while opening a new DCA source. - Dca(DcaError), - /// An error occurred while reading, or opening a file. - Io(IoError), - /// An error occurred while parsing JSON (i.e., during metadata/stereo detection). - Json { - /// Json error - error: JsonError, - /// Text that failed to be parsed - parsed_text: String, - }, - /// An error occurred within the Opus codec. - Opus(OpusError), - /// Failed to extract metadata from alternate pipe. - Metadata, - /// Apparently failed to create stdout. - Stdout, - /// An error occurred while checking if a path is stereo. - Streams, - /// Configuration error for a cached Input. - Streamcatcher(CatcherError), - /// An error occurred while processing the JSON output from `youtube-dl`. - /// - /// The JSON output is given. - YouTubeDlProcessing(Value), - /// An error occurred while running `youtube-dl`. - YouTubeDlRun(Output), - /// The `url` field of the `youtube-dl` JSON output was not present. - /// - /// The JSON output is given. - YouTubeDlUrl(Value), -} - -impl From for Error { - fn from(e: CatcherError) -> Self { - Error::Streamcatcher(e) - } -} - -impl From for Error { - fn from(e: DcaError) -> Self { - Error::Dca(e) - } -} - -impl From for Error { - fn from(e: IoError) -> Error { - Error::Io(e) - } -} - -impl From for Error { - fn from(e: OpusError) -> Error { - Error::Opus(e) - } -} - -impl fmt::Display for Error { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match self { - Error::Dca(_) => write!(f, "opening file DCA failed"), - Error::Io(e) => e.fmt(f), - Error::Json { - error: _, - parsed_text: _, - } => write!(f, "parsing JSON failed"), - Error::Opus(e) => e.fmt(f), - Error::Metadata => write!(f, "extracting metadata failed"), - Error::Stdout => write!(f, "creating stdout failed"), - Error::Streams => write!(f, "checking if path is stereo failed"), - Error::Streamcatcher(_) => write!(f, "invalid config for cached input"), - Error::YouTubeDlProcessing(_) => write!(f, "youtube-dl returned invalid JSON"), - Error::YouTubeDlRun(o) => write!(f, "youtube-dl encontered an error: {:?}", o), - Error::YouTubeDlUrl(_) => write!(f, "missing youtube-dl url"), - } - } -} - -impl StdError for Error { - fn source(&self) -> Option<&(dyn StdError + 'static)> { - match self { - Error::Dca(e) => Some(e), - Error::Io(e) => e.source(), - Error::Json { - error, - parsed_text: _, - } => Some(error), - Error::Opus(e) => e.source(), - Error::Metadata => None, - Error::Stdout => None, - Error::Streams => None, - Error::Streamcatcher(e) => Some(e), - Error::YouTubeDlProcessing(_) => None, - Error::YouTubeDlRun(_) => None, - Error::YouTubeDlUrl(_) => None, - } - } -} - -/// An error returned from the [`dca`] method. -/// -/// [`dca`]: crate::input::dca #[derive(Debug)] -#[non_exhaustive] -pub enum DcaError { - /// An error occurred while reading, or opening a file. - IoError(IoError), - /// The file opened did not have a valid DCA JSON header. - InvalidHeader, - /// The file's metadata block was invalid, or could not be parsed. - InvalidMetadata(JsonError), - /// The file's header reported an invalid metadata block size. - InvalidSize(i32), - /// An error was encountered while creating a new Opus decoder. - Opus(OpusError), +pub enum AudioStreamError { + /// The operation failed, and should be retried after a given time. + /// + /// Create operations invoked by the driver will retry on the first tick + /// after this time has passed. + RetryIn(Duration), + /// The operation failed, and should not be retried. + Fail(Box), + /// The operation was not supported, and will never succeed. + Unsupported, } -impl fmt::Display for DcaError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { +impl Display for AudioStreamError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("failed to create audio: ")?; match self { - DcaError::IoError(e) => e.fmt(f), - DcaError::InvalidHeader => write!(f, "invalid header"), - DcaError::InvalidMetadata(_) => write!(f, "invalid metadata"), - DcaError::InvalidSize(e) => write!(f, "invalid metadata block size: {}", e), - DcaError::Opus(e) => e.fmt(f), + Self::RetryIn(t) => f.write_fmt(format_args!("retry in {:.2}s", t.as_secs_f32())), + Self::Fail(why) => f.write_fmt(format_args!("{}", why)), + Self::Unsupported => f.write_str("operation was not supported"), } } } -impl StdError for DcaError { - fn source(&self) -> Option<&(dyn StdError + 'static)> { - match self { - DcaError::IoError(e) => e.source(), - DcaError::InvalidHeader => None, - DcaError::InvalidMetadata(e) => Some(e), - DcaError::InvalidSize(_) => None, - DcaError::Opus(e) => e.source(), - } +impl Error for AudioStreamError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + None } } -/// Convenience type for fallible return of [`Input`]s. +/// Errors encountered when readying or pre-processing an [`Input`]. /// -/// [`Input`]: crate::input::Input -pub type Result = std::result::Result; +/// [`Input`]: super::Input +#[non_exhaustive] +#[derive(Debug)] +pub enum MakePlayableError { + /// Failed to create a [`LiveInput`] from the lazy [`Compose`]. + /// + /// [`LiveInput`]: super::LiveInput + /// [`Compose`]: super::Compose + Create(AudioStreamError), + /// Failed to read headers, codecs, or a valid stream from a [`LiveInput`]. + /// + /// [`LiveInput`]: super::LiveInput + Parse(SymphError), + /// A blocking thread panicked or failed to return a parsed input. + Panicked, +} + +impl Display for MakePlayableError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("failed to make track playable: ")?; + match self { + Self::Create(c) => { + f.write_str("input creation [")?; + f.write_fmt(format_args!("{}", &c))?; + f.write_str("]") + }, + Self::Parse(p) => { + f.write_str("parsing formats/codecs [")?; + f.write_fmt(format_args!("{}", &p))?; + f.write_str("]") + }, + Self::Panicked => f.write_str("panic during blocking I/O in parse"), + } + } +} + +impl Error for MakePlayableError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + None + } +} + +impl From for MakePlayableError { + fn from(val: AudioStreamError) -> Self { + Self::Create(val) + } +} + +impl From for MakePlayableError { + fn from(val: SymphError) -> Self { + Self::Parse(val) + } +} + +/// Errors encountered when trying to access in-stream [`Metadata`] for an [`Input`]. +/// +/// Both cases can be solved by using [`Input::make_playable`] or [`LiveInput::promote`]. +/// +/// [`Input`]: super::Input +/// [`Metadata`]: super::Metadata +/// [`Input::make_playable`]: super::Input::make_playable +/// [`LiveInput::promote`]: super::LiveInput::promote +#[non_exhaustive] +#[derive(Debug)] +pub enum MetadataError { + /// This input is currently lazily initialised, and must be made live. + NotLive, + /// This input is ready, but has not had its headers parsed. + NotParsed, +} + +impl Display for MetadataError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("failed to get metadata: ")?; + match self { + Self::NotLive => f.write_str("the input is not live, and hasn't been parsed"), + Self::NotParsed => f.write_str("the input is live but hasn't been parsed"), + } + } +} + +impl Error for MetadataError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + None + } +} + +/// Errors encountered when trying to access out-of-band [`AuxMetadata`] for an [`Input`] +/// or [`Compose`]. +/// +/// [`Input`]: super::Input +/// [`AuxMetadata`]: super::AuxMetadata +/// [`Compose`]: super::Compose +#[non_exhaustive] +#[derive(Debug)] +pub enum AuxMetadataError { + /// This input has no lazy [`Compose`] initialiser, which is needed to + /// retrieve [`AuxMetadata`]. + /// + /// [`Compose`]: super::Compose + /// [`AuxMetadata`]: super::AuxMetadata + NoCompose, + /// There was an error when trying to access auxiliary metadata. + Retrieve(AudioStreamError), +} + +impl Display for AuxMetadataError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("failed to get aux_metadata: ")?; + match self { + Self::NoCompose => f.write_str("the input has no Compose object"), + Self::Retrieve(e) => + f.write_fmt(format_args!("aux_metadata error from Compose: {}", e)), + } + } +} + +impl Error for AuxMetadataError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + None + } +} + +impl From for AuxMetadataError { + fn from(val: AudioStreamError) -> Self { + Self::Retrieve(val) + } +} diff --git a/src/input/ffmpeg_src.rs b/src/input/ffmpeg_src.rs deleted file mode 100644 index a8e73ca..0000000 --- a/src/input/ffmpeg_src.rs +++ /dev/null @@ -1,159 +0,0 @@ -use super::{ - children_to_reader, - error::{Error, Result}, - Codec, - Container, - Input, - Metadata, -}; -use serde_json::Value; -use std::{ - ffi::OsStr, - process::{Command, Stdio}, -}; -use tokio::process::Command as TokioCommand; -use tracing::debug; - -/// Opens an audio file through `ffmpeg` and creates an audio source. -/// -/// This source is not seek-compatible. -/// If you need looping or track seeking, then consider using -/// [`Restartable::ffmpeg`]. -/// -/// [`Restartable::ffmpeg`]: crate::input::restartable::Restartable::ffmpeg -pub async fn ffmpeg>(path: P) -> Result { - _ffmpeg(path.as_ref()).await -} - -pub(crate) async fn _ffmpeg(path: &OsStr) -> Result { - // Will fail if the path is not to a file on the fs. Likely a YouTube URI. - let is_stereo = is_stereo(path) - .await - .unwrap_or_else(|_e| (false, Default::default())); - let stereo_val = if is_stereo.0 { "2" } else { "1" }; - - _ffmpeg_optioned( - path, - &[], - &[ - "-f", - "s16le", - "-ac", - stereo_val, - "-ar", - "48000", - "-acodec", - "pcm_f32le", - "-", - ], - Some(is_stereo), - ) - .await -} - -/// Opens an audio file through `ffmpeg` and creates an audio source, with -/// user-specified arguments to pass to ffmpeg. -/// -/// Note that this does _not_ build on the arguments passed by the [`ffmpeg`] -/// function. -/// -/// # Examples -/// -/// Pass options to create a custom ffmpeg streamer: -/// -/// ```rust,no_run -/// use songbird::input; -/// -/// let stereo_val = "2"; -/// -/// let streamer = futures::executor::block_on(input::ffmpeg_optioned("./some_file.mp3", &[], &[ -/// "-f", -/// "s16le", -/// "-ac", -/// stereo_val, -/// "-ar", -/// "48000", -/// "-acodec", -/// "pcm_s16le", -/// "-", -/// ])); -/// ``` -/// -/// [`ffmpeg`]: ffmpeg -pub async fn ffmpeg_optioned>( - path: P, - pre_input_args: &[&str], - args: &[&str], -) -> Result { - _ffmpeg_optioned(path.as_ref(), pre_input_args, args, None).await -} - -pub(crate) async fn _ffmpeg_optioned( - path: &OsStr, - pre_input_args: &[&str], - args: &[&str], - is_stereo_known: Option<(bool, Metadata)>, -) -> Result { - let (is_stereo, metadata) = if let Some(vals) = is_stereo_known { - vals - } else { - is_stereo(path) - .await - .ok() - .unwrap_or_else(|| (false, Default::default())) - }; - - let command = Command::new("ffmpeg") - .args(pre_input_args) - .arg("-i") - .arg(path) - .args(args) - .stderr(Stdio::null()) - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .spawn()?; - - Ok(Input::new( - is_stereo, - children_to_reader::(vec![command]), - Codec::FloatPcm, - Container::Raw, - Some(metadata), - )) -} - -pub(crate) async fn is_stereo(path: &OsStr) -> Result<(bool, Metadata)> { - let args = [ - "-v", - "quiet", - "-of", - "json", - "-show_format", - "-show_streams", - "-i", - ]; - - let out = TokioCommand::new("ffprobe") - .args(&args) - .arg(path) - .stdin(Stdio::null()) - .output() - .await?; - - let value: Value = serde_json::from_reader(&out.stdout[..]).map_err(|err| Error::Json { - error: err, - parsed_text: std::str::from_utf8(&out.stdout[..]) - .unwrap_or_default() - .to_string(), - })?; - - let metadata = Metadata::from_ffprobe_json(&value); - - debug!("FFprobe metadata {:?}", metadata); - - if let Some(count) = metadata.channels { - Ok((count == 2, metadata)) - } else { - Err(Error::Streams) - } -} diff --git a/src/input/input_tests.rs b/src/input/input_tests.rs new file mode 100644 index 0000000..d4d3999 --- /dev/null +++ b/src/input/input_tests.rs @@ -0,0 +1,147 @@ +use crate::{ + driver::Driver, + tracks::{PlayMode, ReadyState, Track}, + Config, +}; + +use std::time::Duration; + +pub async fn track_plays_passthrough(make_track: F) +where + T: Into, + F: FnOnce() -> T, +{ + track_plays_base(make_track, true, None).await; +} + +pub async fn track_plays_passthrough_when_is_only_active(make_track: F) +where + T: Into, + F: FnOnce() -> T, +{ + track_plays_base( + make_track, + true, + Some(include_bytes!("../../resources/loop.wav")), + ) + .await; +} + +pub async fn track_plays_mixed(make_track: F) +where + T: Into, + F: FnOnce() -> T, +{ + track_plays_base(make_track, false, None).await; +} + +pub async fn track_plays_base( + make_track: F, + passthrough: bool, + dummy_track: Option<&'static [u8]>, +) where + T: Into, + F: FnOnce() -> T, +{ + let (t_handle, config) = Config::test_cfg(true); + let mut driver = Driver::new(config.clone()); + + // Used to ensure that paused tracks won't prevent passthrough from happening + // i.e., most queue users :) + if let Some(audio_data) = dummy_track { + driver.play(Track::from(audio_data).pause()); + } + + let file = make_track(); + + // Get input in place, playing. Wait for IO to ready. + t_handle.ready_track(&driver.play(file.into()), None).await; + t_handle.tick(1); + + // post-conditions: + // 1) track produces a packet. + // 2) that packet is passthrough. + let pkt = t_handle.recv_async().await; + let pkt = pkt.raw().unwrap(); + + if passthrough { + assert!(pkt.is_passthrough()); + } else { + assert!(pkt.is_mixed_with_nonzero_signal()); + } +} + +pub async fn forward_seek_correct(make_track: F) +where + T: Into, + F: FnOnce() -> T, +{ + let (t_handle, config) = Config::test_cfg(true); + let mut driver = Driver::new(config.clone()); + + let file = make_track(); + let handle = driver.play(file.into()); + + // Get input in place, playing. Wait for IO to ready. + t_handle.ready_track(&handle, None).await; + + let target_time = Duration::from_secs(30); + assert!(!handle.seek(target_time).is_hung_up()); + t_handle.ready_track(&handle, None).await; + + // post-conditions: + // 1) track is readied + // 2) track's position is approx 30s + // 3) track's play time is considerably less (O(5s)) + let state = handle.get_info(); + t_handle.spawn_ticker().await; + let state = state.await.expect("Should have received valid state."); + + assert_eq!(state.ready, ReadyState::Playable); + assert_eq!(state.playing, PlayMode::Play); + assert!(state.play_time < Duration::from_secs(5)); + assert!( + state.position < target_time + Duration::from_millis(100) + && state.position > target_time - Duration::from_millis(100) + ); +} + +pub async fn backward_seek_correct(make_track: F) +where + T: Into, + F: FnOnce() -> T, +{ + let (t_handle, config) = Config::test_cfg(true); + let mut driver = Driver::new(config.clone()); + + let file = make_track(); + let handle = driver.play(file.into()); + + // Get input in place, playing. Wait for IO to ready. + t_handle.ready_track(&handle, None).await; + + // Accelerated playout -- 4 seconds worth. + let n_secs = 4; + let n_ticks = 50 * n_secs; + t_handle.skip(n_ticks).await; + + let target_time = Duration::from_secs(1); + assert!(!handle.seek(target_time).is_hung_up()); + t_handle.ready_track(&handle, None).await; + + // post-conditions: + // 1) track is readied + // 2) track's position is approx 1s + // 3) track's play time is preserved (About 4s) + let state = handle.get_info(); + t_handle.spawn_ticker().await; + let state = state.await.expect("Should have received valid state."); + + assert_eq!(state.ready, ReadyState::Playable); + assert_eq!(state.playing, PlayMode::Play); + assert!(state.play_time >= Duration::from_secs(n_secs)); + assert!( + state.position < target_time + Duration::from_millis(100) + && state.position > target_time - Duration::from_millis(100) + ); +} diff --git a/src/input/live_input.rs b/src/input/live_input.rs new file mode 100644 index 0000000..a44166a --- /dev/null +++ b/src/input/live_input.rs @@ -0,0 +1,147 @@ +use super::{AudioStream, Metadata, MetadataError, Parsed}; + +use symphonia_core::{ + codecs::{CodecRegistry, Decoder, DecoderOptions}, + errors::Error as SymphError, + formats::FormatOptions, + io::{MediaSource, MediaSourceStream, MediaSourceStreamOptions}, + meta::MetadataOptions, + probe::Probe, +}; + +/// An initialised audio source. +/// +/// This type's variants reflect files at different stages of readiness for use by +/// symphonia. [`Parsed`] file streams are ready for playback. +/// +/// [`Parsed`]: Self::Parsed +pub enum LiveInput { + /// An unread, raw file stream. + Raw(AudioStream>), + /// An unread file which has been wrapped with a large read-ahead buffer. + Wrapped(AudioStream), + /// An audio file which has had its headers parsed and decoder state built. + Parsed(Parsed), +} + +impl LiveInput { + /// Converts this audio source into a [`Parsed`] object using the supplied format and codec + /// registries. + /// + /// Where applicable, this will convert [`Raw`] -> [`Wrapped`] -> [`Parsed`], and will + /// play the default track (or the first encountered track if this is not available) if a + /// container holds multiple audio streams. + /// + /// *This is a blocking operation. Symphonia uses standard library I/O (e.g., [`Read`], [`Seek`]). + /// If you wish to use this from an async task, you must do so within `spawn_blocking`.* + /// + /// [`Parsed`]: Self::Parsed + /// [`Raw`]: Self::Raw + /// [`Wrapped`]: Self::Wrapped + /// [`Read`]: https://doc.rust-lang.org/std/io/trait.Read.html + /// [`Seek`]: https://doc.rust-lang.org/std/io/trait.Seek.html + pub fn promote(self, codecs: &CodecRegistry, probe: &Probe) -> Result { + let mut out = self; + + if let LiveInput::Raw(r) = out { + // TODO: allow passing in of MSS options? + let mss = MediaSourceStream::new(r.input, MediaSourceStreamOptions::default()); + out = LiveInput::Wrapped(AudioStream { + input: mss, + hint: r.hint, + }); + } + + if let LiveInput::Wrapped(w) = out { + let hint = w.hint.unwrap_or_default(); + let input = w.input; + let supports_backseek = input.is_seekable(); + + let probe_data = probe.format( + &hint, + input, + &FormatOptions::default(), + &MetadataOptions::default(), + )?; + let format = probe_data.format; + let meta = probe_data.metadata; + + let mut default_track_id = format.default_track().map(|track| track.id); + let mut decoder: Option> = None; + + // Awkward loop: we need BOTH a track ID, and a decoder matching that track ID. + // Take default track (if it exists), take first track to be found otherwise. + for track in format.tracks() { + if default_track_id.is_some() && Some(track.id) != default_track_id { + continue; + } + + let this_decoder = codecs.make(&track.codec_params, &DecoderOptions::default())?; + + decoder = Some(this_decoder); + default_track_id = Some(track.id); + + break; + } + + // No tracks is a playout error, a bad default track is also possible. + // These are probably malformed? We could go best-effort, and fall back to tracks[0] + // but drop such tracks for now. + let track_id = default_track_id.ok_or(SymphError::DecodeError("no track found"))?; + let decoder = decoder.ok_or(SymphError::DecodeError( + "reported default track did not exist", + ))?; + + let p = Parsed { + format, + decoder, + track_id, + meta, + supports_backseek, + }; + + out = LiveInput::Parsed(p); + } + + Ok(out) + } + + /// Returns a reference to the data parsed from this input stream, if it has + /// been made available via [`Self::promote`]. + #[must_use] + pub fn parsed(&self) -> Option<&Parsed> { + if let Self::Parsed(parsed) = self { + Some(parsed) + } else { + None + } + } + + /// Returns a mutable reference to the data parsed from this input stream, if it + /// has been made available via [`Self::promote`]. + pub fn parsed_mut(&mut self) -> Option<&mut Parsed> { + if let Self::Parsed(parsed) = self { + Some(parsed) + } else { + None + } + } + + /// Returns whether this stream's headers have been fully parsed, and so whether + /// the track can be played or have its metadata read. + #[must_use] + pub fn is_playable(&self) -> bool { + self.parsed().is_some() + } + + /// Tries to get any information about this audio stream acquired during parsing. + /// + /// Only exists when this input is [`LiveInput::Parsed`]. + pub fn metadata(&mut self) -> Result { + if let Some(parsed) = self.parsed_mut() { + Ok(parsed.into()) + } else { + Err(MetadataError::NotParsed) + } + } +} diff --git a/src/input/metadata.rs b/src/input/metadata.rs deleted file mode 100644 index f05f81c..0000000 --- a/src/input/metadata.rs +++ /dev/null @@ -1,198 +0,0 @@ -use crate::constants::*; -use serde_json::Value; -use std::time::Duration; - -/// Information about an [`Input`] source. -/// -/// [`Input`]: crate::input::Input -#[derive(Clone, Debug, Default, Eq, PartialEq)] -pub struct Metadata { - /// The track of this stream. - pub track: Option, - /// The main artist of this stream. - pub artist: Option, - /// The date of creation of this stream. - pub date: Option, - - /// The number of audio channels in this stream. - /// - /// Any number `>= 2` is treated as stereo. - pub channels: Option, - /// The YouTube channel of this stream. - pub channel: Option, - /// The time at which the first true sample is played back. - /// - /// This occurs as an artefact of coder delay. - pub start_time: Option, - /// The reported duration of this stream. - pub duration: Option, - /// The sample rate of this stream. - pub sample_rate: Option, - /// The source url of this stream. - pub source_url: Option, - /// The YouTube title of this stream. - pub title: Option, - /// The thumbnail url of this stream. - pub thumbnail: Option, -} - -impl Metadata { - /// Extract metadata and details from the output of - /// `ffprobe`. - pub fn from_ffprobe_json(value: &Value) -> Self { - let format = value.as_object().and_then(|m| m.get("format")); - - let duration = format - .and_then(|m| m.get("duration")) - .and_then(Value::as_str) - .and_then(|v| v.parse::().ok()) - .map(Duration::from_secs_f64); - - let start_time = format - .and_then(|m| m.get("start_time")) - .and_then(Value::as_str) - .and_then(|v| v.parse::().ok().map(|t| t.max(0.0))) - .map(Duration::from_secs_f64); - - let tags = format.and_then(|m| m.get("tags")); - - let track = tags - .and_then(|m| m.get("title")) - .and_then(Value::as_str) - .map(str::to_string); - - let artist = tags - .and_then(|m| m.get("artist")) - .and_then(Value::as_str) - .map(str::to_string); - - let date = tags - .and_then(|m| m.get("date")) - .and_then(Value::as_str) - .map(str::to_string); - - let stream = value - .as_object() - .and_then(|m| m.get("streams")) - .and_then(|v| v.as_array()) - .and_then(|v| { - v.iter() - .find(|line| line.get("codec_type").and_then(Value::as_str) == Some("audio")) - }); - - let channels = stream - .and_then(|m| m.get("channels")) - .and_then(Value::as_u64) - .map(|v| v as u8); - - let sample_rate = stream - .and_then(|m| m.get("sample_rate")) - .and_then(Value::as_str) - .and_then(|v| v.parse::().ok()) - .map(|v| v as u32); - - Self { - track, - artist, - date, - - channels, - start_time, - duration, - sample_rate, - - ..Default::default() - } - } - - /// Use `youtube-dl`'s JSON output for metadata for an online resource. - pub fn from_ytdl_output(value: Value) -> Self { - let obj = value.as_object(); - - let track = obj - .and_then(|m| m.get("track")) - .and_then(Value::as_str) - .map(str::to_string); - - let true_artist = obj - .and_then(|m| m.get("artist")) - .and_then(Value::as_str) - .map(str::to_string); - - let artist = true_artist.or_else(|| { - obj.and_then(|m| m.get("uploader")) - .and_then(Value::as_str) - .map(str::to_string) - }); - - let r_date = obj - .and_then(|m| m.get("release_date")) - .and_then(Value::as_str) - .map(str::to_string); - - let date = r_date.or_else(|| { - obj.and_then(|m| m.get("upload_date")) - .and_then(Value::as_str) - .map(str::to_string) - }); - - let channel = obj - .and_then(|m| m.get("channel")) - .and_then(Value::as_str) - .map(str::to_string); - - let duration = obj - .and_then(|m| m.get("duration")) - .and_then(Value::as_f64) - .map(Duration::from_secs_f64); - - let source_url = obj - .and_then(|m| m.get("webpage_url")) - .and_then(Value::as_str) - .map(str::to_string); - - let title = obj - .and_then(|m| m.get("title")) - .and_then(Value::as_str) - .map(str::to_string); - - let thumbnail = obj - .and_then(|m| m.get("thumbnail")) - .and_then(Value::as_str) - .map(str::to_string); - - Self { - track, - artist, - date, - - channels: Some(2), - channel, - duration, - sample_rate: Some(SAMPLE_RATE_RAW as u32), - source_url, - title, - thumbnail, - - ..Default::default() - } - } - - /// Move all fields from a `Metadata` object into a new one. - pub fn take(&mut self) -> Self { - Self { - track: self.track.take(), - artist: self.artist.take(), - date: self.date.take(), - - channels: self.channels.take(), - channel: self.channel.take(), - start_time: self.start_time.take(), - duration: self.duration.take(), - sample_rate: self.sample_rate.take(), - source_url: self.source_url.take(), - title: self.title.take(), - thumbnail: self.thumbnail.take(), - } - } -} diff --git a/src/input/metadata/ffprobe.rs b/src/input/metadata/ffprobe.rs new file mode 100644 index 0000000..728034a --- /dev/null +++ b/src/input/metadata/ffprobe.rs @@ -0,0 +1,173 @@ +use super::AuxMetadata; +use serde::{Deserialize, Serialize}; +use serde_aux::prelude::*; +use std::{collections::HashMap, time::Duration}; + +// These have been put together by looking at ffprobe's output +// and the canonical data formats given in +// https://github.com/FFmpeg/FFmpeg/blob/master/doc/ffprobe.xsd + +#[derive(Deserialize, Serialize)] +pub struct Output { + pub streams: Vec, + pub format: Format, +} + +#[derive(Deserialize, Serialize)] +pub struct Stream { + pub index: u64, + pub codec_name: Option, + pub codec_long_name: Option, + pub profile: Option, + pub codec_type: Option, + pub codec_tag: String, + pub codec_tag_string: String, + pub extradata: Option, + pub extradata_size: Option, + pub extradata_hash: Option, + + // Video attributes skipped. + pub sample_fmt: Option, + #[serde(deserialize_with = "deserialize_option_number_from_string")] + pub sample_rate: Option, + pub channels: Option, + pub channel_layout: Option, + pub bits_per_sample: Option, + + pub id: Option, + pub r_frame_rate: String, + pub avg_frame_rate: String, + pub time_base: String, + pub start_pts: Option, + #[serde(deserialize_with = "deserialize_option_number_from_string")] + pub start_time: Option, + pub duration_ts: Option, + #[serde(deserialize_with = "deserialize_option_number_from_string")] + pub duration: Option, + #[serde(deserialize_with = "deserialize_option_number_from_string")] + pub bit_rate: Option, + #[serde(deserialize_with = "deserialize_option_number_from_string")] + pub max_bit_rate: Option, + pub bits_per_raw_sample: Option, + pub nb_frames: Option, + pub nb_read_frames: Option, + pub nb_read_packets: Option, + + // Side Data List skipped. + pub disposition: Option, + pub tags: Option>, +} + +#[allow(clippy::struct_excessive_bools)] +#[derive(Deserialize, Serialize)] +pub struct Disposition { + #[serde(deserialize_with = "deserialize_bool_from_anything")] + pub default: bool, + #[serde(deserialize_with = "deserialize_bool_from_anything")] + pub dub: bool, + #[serde(deserialize_with = "deserialize_bool_from_anything")] + pub original: bool, + #[serde(deserialize_with = "deserialize_bool_from_anything")] + pub comment: bool, + #[serde(deserialize_with = "deserialize_bool_from_anything")] + pub lyrics: bool, + #[serde(deserialize_with = "deserialize_bool_from_anything")] + pub karaoke: bool, + #[serde(deserialize_with = "deserialize_bool_from_anything")] + pub forced: bool, + #[serde(deserialize_with = "deserialize_bool_from_anything")] + pub hearing_impaired: bool, + #[serde(deserialize_with = "deserialize_bool_from_anything")] + pub visual_impaired: bool, + #[serde(deserialize_with = "deserialize_bool_from_anything")] + pub clean_effects: bool, + #[serde(deserialize_with = "deserialize_bool_from_anything")] + pub attached_pic: bool, + #[serde(deserialize_with = "deserialize_bool_from_anything")] + pub timed_thumbnails: bool, + #[serde(deserialize_with = "deserialize_bool_from_anything")] + pub captions: bool, + #[serde(deserialize_with = "deserialize_bool_from_anything")] + pub descriptions: bool, + #[serde(deserialize_with = "deserialize_bool_from_anything")] + pub metadata: bool, + #[serde(deserialize_with = "deserialize_bool_from_anything")] + pub dependent: bool, + #[serde(deserialize_with = "deserialize_bool_from_anything")] + pub still_image: bool, +} + +#[derive(Deserialize, Serialize)] +pub struct Format { + pub filename: String, + pub nb_streams: u64, + pub nb_programs: u64, + pub format_name: String, + pub format_long_name: Option, + + #[serde(deserialize_with = "deserialize_option_number_from_string")] + pub start_time: Option, + #[serde(deserialize_with = "deserialize_option_number_from_string")] + pub duration: Option, + #[serde(deserialize_with = "deserialize_option_number_from_string")] + pub size: Option, + #[serde(deserialize_with = "deserialize_option_number_from_string")] + pub bit_rate: Option, + + pub probe_score: i64, + pub tags: Option>, +} + +fn apply_tags(tag_map: HashMap, dest: &mut AuxMetadata) { + for (k, v) in tag_map { + match k.as_str().to_lowercase().as_str() { + "title" => dest.title = Some(v), + "album" => dest.album = Some(v), + "artist" => dest.artist = Some(v), + "date" => dest.date = Some(v), + "channels" => + if let Ok(chans) = str::parse::(&v) { + dest.channels = Some(chans); + }, + "sample_rate" => + if let Ok(samples) = str::parse::(&v) { + dest.sample_rate = Some(samples); + }, + _ => {}, + } + } +} + +impl Output { + pub fn into_aux_metadata(self) -> AuxMetadata { + let duration = self.format.duration.map(Duration::from_secs_f64); + let start_time = self + .format + .duration + .map(|v| v.max(0.0)) + .map(Duration::from_secs_f64); + + let mut out = AuxMetadata { + start_time, + duration, + + ..AuxMetadata::default() + }; + + if let Some(tags) = self.format.tags { + apply_tags(tags, &mut out); + } + + for stream in self.streams { + if stream.codec_type.as_deref() != Some("audio") { + continue; + } + + if let Some(tags) = stream.tags { + apply_tags(tags, &mut out); + } + } + + out + } +} diff --git a/src/input/metadata/mod.rs b/src/input/metadata/mod.rs new file mode 100644 index 0000000..916ac61 --- /dev/null +++ b/src/input/metadata/mod.rs @@ -0,0 +1,109 @@ +use std::time::Duration; +use symphonia_core::{meta::Metadata as ContainerMetadata, probe::ProbedMetadata}; + +pub(crate) mod ffprobe; +pub(crate) mod ytdl; + +use super::Parsed; + +/// Extra information about an [`Input`] which is acquired without +/// parsing the file itself (e.g., from a webpage). +/// +/// You can access this via [`Input::aux_metadata`] and [`Compose::aux_metadata`]. +/// +/// [`Input`]: crate::input::Input +/// [`Input::aux_metadata`]: crate::input::Input::aux_metadata +/// [`Compose::aux_metadata`]: crate::input::Compose::aux_metadata +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct AuxMetadata { + /// The track name of this stream. + pub track: Option, + /// The main artist of this stream. + pub artist: Option, + /// The album name of this stream. + pub album: Option, + /// The date of creation of this stream. + pub date: Option, + + /// The number of audio channels in this stream. + pub channels: Option, + /// The YouTube channel of this stream. + pub channel: Option, + /// The time at which the first true sample is played back. + /// + /// This occurs as an artefact of coder delay. + pub start_time: Option, + /// The reported duration of this stream. + pub duration: Option, + /// The sample rate of this stream. + pub sample_rate: Option, + /// The source url of this stream. + pub source_url: Option, + /// The YouTube title of this stream. + pub title: Option, + /// The thumbnail url of this stream. + pub thumbnail: Option, +} + +impl AuxMetadata { + /// Extract metadata and details from the output of `ffprobe -of json`. + pub fn from_ffprobe_json(value: &[u8]) -> Result { + let output: ffprobe::Output = serde_json::from_slice(value)?; + + Ok(output.into_aux_metadata()) + } + + /// Move all fields from an [`AuxMetadata`] object into a new one. + #[must_use] + pub fn take(&mut self) -> Self { + Self { + track: self.track.take(), + artist: self.artist.take(), + album: self.album.take(), + date: self.date.take(), + channels: self.channels.take(), + channel: self.channel.take(), + start_time: self.start_time.take(), + duration: self.duration.take(), + sample_rate: self.sample_rate.take(), + source_url: self.source_url.take(), + title: self.title.take(), + thumbnail: self.thumbnail.take(), + } + } +} + +/// In-stream information about an [`Input`] acquired by parsing an audio file. +/// +/// To access this, the [`Input`] must be made live and parsed by symphonia. To do +/// this, you can: +/// * Pre-process the track in your own code using [`Input::make_playable`], and +/// then [`Input::metadata`]. +/// * Use [`TrackHandle::action`] to access the track's metadata via [`View`], +/// *if the track has started or been made playable*. +/// +/// You probably want to use [`AuxMetadata`] instead; this requires a live track, +/// which has higher memory use for buffers etc. +/// +/// [`Input`]: crate::input::Input +/// [`Input::make_playable`]: super::Input::make_playable +/// [`Input::metadata`]: super::Input::metadata +/// [`TrackHandle::action`]: crate::tracks::TrackHandle::action +/// [`View`]: crate::tracks::View +pub struct Metadata<'a> { + /// Metadata found while probing for the format of an [`Input`] (e.g., ID3 tags). + /// + /// [`Input`]: crate::input::Input + pub probe: &'a mut ProbedMetadata, + /// Metadata found inside the format/container of an audio stream. + pub format: ContainerMetadata<'a>, +} + +impl<'a> From<&'a mut Parsed> for Metadata<'a> { + fn from(val: &'a mut Parsed) -> Self { + Metadata { + probe: &mut val.meta, + format: val.format.metadata(), + } + } +} diff --git a/src/input/metadata/ytdl.rs b/src/input/metadata/ytdl.rs new file mode 100644 index 0000000..8d34ab6 --- /dev/null +++ b/src/input/metadata/ytdl.rs @@ -0,0 +1,55 @@ +use super::AuxMetadata; +use crate::constants::SAMPLE_RATE_RAW; +use serde::{Deserialize, Serialize}; +use std::{collections::HashMap, time::Duration}; + +#[derive(Deserialize, Serialize)] +pub struct Output { + pub artist: Option, + pub album: Option, + pub channel: Option, + pub duration: Option, + pub filesize: Option, + pub http_headers: Option>, + pub release_date: Option, + pub thumbnail: Option, + pub title: Option, + pub track: Option, + pub upload_date: Option, + pub uploader: Option, + pub url: String, + pub webpage_url: Option, +} + +impl Output { + pub fn as_aux_metadata(&self) -> AuxMetadata { + let album = self.album.clone(); + let track = self.track.clone(); + let true_artist = self.artist.as_ref(); + let artist = true_artist.or(self.uploader.as_ref()).cloned(); + let r_date = self.release_date.as_ref(); + let date = r_date.or(self.upload_date.as_ref()).cloned(); + let channel = self.channel.clone(); + let duration = self.duration.map(Duration::from_secs_f64); + let source_url = self.webpage_url.clone(); + let title = self.title.clone(); + let thumbnail = self.thumbnail.clone(); + + AuxMetadata { + track, + artist, + album, + date, + + channels: Some(2), + channel, + duration, + sample_rate: Some(SAMPLE_RATE_RAW as u32), + source_url, + title, + thumbnail, + + ..AuxMetadata::default() + } + } +} diff --git a/src/input/mod.rs b/src/input/mod.rs index af3e4c5..2d42288 100644 --- a/src/input/mod.rs +++ b/src/input/mod.rs @@ -1,594 +1,382 @@ //! Raw audio input data streams and sources. //! -//! [`Input`] is handled in Songbird by combining metadata with: -//! * A 48kHz audio bytestream, via [`Reader`], -//! * A [`Container`] describing the framing mechanism of the bytestream, -//! * A [`Codec`], defining the format of audio frames. +//! [`Input`]s in Songbird are based on [symphonia], which provides demuxing, +//! decoding and management of synchronous byte sources (i.e., any items which +//! `impl` [`Read`]). //! -//! When used as a [`Read`], the output bytestream will be a floating-point -//! PCM stream at 48kHz, matching the channel count of the input source. +//! Songbird adds support for the Opus codec to symphonia via [`OpusDecoder`], +//! the [DCA1] file format via [`DcaReader`], and a simple PCM adapter via [`RawReader`]; +//! the [format] and [codec registries] in [`codecs`] install these on top of those +//! enabled in your `Cargo.toml` when you include symphonia. +//! +//! ## Common sources +//! * Any owned byte slice: `&'static [u8]`, `Bytes`, or `Vec`, +//! * [`File`] offers a lazy way to open local audio files, +//! * [`HttpRequest`] streams a given file from a URL using the reqwest HTTP library, +//! * [`YoutubeDl`] uses `yt-dlp` (or any other `youtube-dl`-like program) to scrape +//! a target URL for a usable audio stream, before opening an [`HttpRequest`]. +//! +//! ## Adapters +//! Songbird includes several adapters to make developing your own inputs easier: +//! * [`cached::*`], which allow seeking and shared caching of an input stream (storing +//! it in memory in a variety of formats), +//! * [`ChildContainer`] for managing audio given by a process chain, +//! * [`RawAdapter`], for feeding in a synchronous `f32`-PCM stream, and +//! * [`AsyncAdapterStream`], for passing bytes from an `AsyncRead` (`+ AsyncSeek`) stream +//! into the mixer. //! //! ## Opus frame passthrough. -//! Some sources, such as [`Compressed`] or the output of [`dca`], support +//! Some sources, such as [`Compressed`] or any WebM/Opus/DCA file, support //! direct frame passthrough to the driver. This lets you directly send the //! audio data you have *without decoding, re-encoding, or mixing*. In many -//! cases, this can greatly reduce the processing/compute cost of the driver. +//! cases, this can greatly reduce the CPU cost required by the driver. //! //! This functionality requires that: //! * only one track is active (including paused tracks), //! * that track's input supports direct Opus frame reads, -//! * its [`Input`] [meets the promises described herein](codec/struct.OpusDecoderState.html#structfield.allow_passthrough), +//! * this input's frames are all sized to 20ms. //! * and that track's volume is set to `1.0`. //! -//! [`Input`]: Input -//! [`Reader`]: reader::Reader -//! [`Container`]: Container -//! [`Codec`]: Codec +//! [`Input`]s which are almost suitable but which have **any** illegal frames will be +//! blocked from passthrough to prevent glitches such as repeated encoder frame gaps. +//! +//! [symphonia]: https://docs.rs/symphonia //! [`Read`]: https://doc.rust-lang.org/std/io/trait.Read.html //! [`Compressed`]: cached::Compressed -//! [`dca`]: dca() +//! [DCA1]: https://github.com/bwmarrin/dca +//! [`registry::*`]: registry +//! [`cached::*`]: cached +//! [`OpusDecoder`]: codecs::OpusDecoder +//! [`DcaReader`]: codecs::DcaReader +//! [`RawReader`]: codecs::RawReader +//! [format]: static@codecs::PROBE +//! [codec registries]: static@codecs::CODEC_REGISTRY -pub mod cached; -mod child; -pub mod codec; -mod container; -mod dca; -pub mod error; -mod ffmpeg_src; +mod adapters; +mod audiostream; +pub mod codecs; +mod compose; +mod error; +#[cfg(test)] +pub mod input_tests; +mod live_input; mod metadata; -pub mod reader; -pub mod restartable; +mod parsed; +mod sources; pub mod utils; -mod ytdl_src; pub use self::{ - child::*, - codec::{Codec, CodecType}, - container::{Container, Frame}, - dca::dca, - ffmpeg_src::*, - metadata::Metadata, - reader::Reader, - restartable::Restartable, - ytdl_src::*, + adapters::*, + audiostream::*, + compose::*, + error::*, + live_input::*, + metadata::*, + parsed::*, + sources::*, }; -use crate::constants::*; -use audiopus::coder::GenericCtl; -use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; -use cached::OpusCompressor; -use error::{Error, Result}; -use tokio::runtime::Handle; +pub use symphonia_core as core; -use std::{ - convert::{TryFrom, TryInto}, - io::{ - self, - Error as IoError, - ErrorKind as IoErrorKind, - Read, - Result as IoResult, - Seek, - SeekFrom, - }, - mem, - time::Duration, -}; -use tracing::{debug, error}; +use std::{error::Error, io::Cursor}; +use symphonia_core::{codecs::CodecRegistry, probe::Probe}; +use tokio::runtime::Handle as TokioHandle; -/// Data and metadata needed to correctly parse a [`Reader`]'s audio bytestream. +/// An audio source, which can be live or lazily initialised. /// -/// See the [module root] for more information. +/// This can be created from a wide variety of sources: +/// * Any owned byte slice: `&'static [u8]`, `Bytes`, or `Vec`, +/// * [`File`] offers a lazy way to open local audio files, +/// * [`HttpRequest`] streams a given file from a URL using the reqwest HTTP library, +/// * [`YoutubeDl`] uses `yt-dlp` (or any other `youtube-dl`-like program) to scrape +/// a target URL for a usable audio stream, before opening an [`HttpRequest`]. /// -/// [`Reader`]: Reader -/// [module root]: super -#[derive(Debug)] -pub struct Input { - /// Information about the played source. - pub metadata: Box, - /// Indicates whether `source` is stereo or mono. - pub stereo: bool, - /// Underlying audio data bytestream. - pub reader: Reader, - /// Decoder used to parse the output of `reader`. - pub kind: Codec, - /// Framing strategy needed to identify frames of compressed audio. - pub container: Container, - pos: usize, +/// Any [`Input`] (or struct with `impl Into`) can also be made into a [`Track`] via +/// `From`/`Into`. +/// +/// # Example +/// +/// ``` +/// # use tokio::runtime; +/// # +/// # let basic_rt = runtime::Builder::new_current_thread().enable_io().build().unwrap(); +/// # basic_rt.block_on(async { +/// use songbird::{ +/// driver::Driver, +/// input::{codecs::*, Compose, Input, MetadataError, YoutubeDl}, +/// tracks::Track, +/// }; +/// // Inputs are played using a `Driver`, or `Call`. +/// let mut driver = Driver::new(Default::default()); +/// +/// // Lazy inputs take very little resources, and don't occupy any resources until we +/// // need to play them (by default). +/// let mut lazy = YoutubeDl::new( +/// reqwest::Client::new(), +/// // Referenced under CC BY-NC-SA 3.0 -- https://creativecommons.org/licenses/by-nc-sa/3.0/ +/// "https://cloudkicker.bandcamp.com/track/94-days".to_string(), +/// ); +/// let lazy_c = lazy.clone(); +/// +/// // With sources like `YoutubeDl`, we can get metadata from, e.g., a track's page. +/// let aux_metadata = lazy.aux_metadata().await.unwrap(); +/// assert_eq!(aux_metadata.track, Some("94 Days".to_string())); +/// +/// // Once we pass an `Input` to the `Driver`, we can only remotely control it via +/// // a `TrackHandle`. +/// let handle = driver.play_input(lazy.into()); +/// +/// // We can also modify some of its initial state via `Track`s. +/// let handle = driver.play(Track::from(lazy_c).volume(0.5).pause()); +/// +/// // In-memory sources like `Vec`, or `&'static [u8]` are easy to use, and only take a +/// // little time for the mixer to parse their headers. +/// // You can also use the adapters in `songbird::input::cached::*`to keep a source +/// // from the Internet, HTTP, or a File in-memory *and* share it among calls. +/// let in_memory = include_bytes!("../../resources/ting.mp3"); +/// let mut in_memory_input = in_memory.into(); +/// +/// // This source is live... +/// assert!(matches!(in_memory_input, Input::Live(..))); +/// // ...but not yet playable, and we can't access its `Metadata`. +/// assert!(!in_memory_input.is_playable()); +/// assert!(matches!(in_memory_input.metadata(), Err(MetadataError::NotParsed))); +/// +/// // If we want to inspect metadata (and we can't use AuxMetadata for any reason), we have +/// // to parse the track ourselves. +/// in_memory_input = in_memory_input +/// .make_playable_async(&CODEC_REGISTRY, &PROBE) +/// .await +/// .expect("WAV support is included, and this file is good!"); +/// +/// // Symphonia's metadata can be difficult to use: prefer `AuxMetadata` when you can! +/// use symphonia_core::meta::{StandardTagKey, Value}; +/// let mut metadata = in_memory_input.metadata(); +/// let meta = metadata.as_mut().unwrap(); +/// let mut probed = meta.probe.get().unwrap(); +/// +/// let track_name = probed +/// .current().unwrap() +/// .tags().iter().filter(|v| v.std_key == Some(StandardTagKey::TrackTitle)) +/// .next().unwrap(); +/// if let Value::String(s) = &track_name.value { +/// assert_eq!(s, "Ting!"); +/// } else { panic!() }; +/// +/// // ...and these are played like any other input. +/// let handle = driver.play_input(in_memory_input); +/// # }); +/// ``` +/// +/// [`Track`]: crate::tracks::Track +pub enum Input { + /// A byte source which is not yet initialised. + /// + /// When a parent track is either played or explicitly readied, the inner [`Compose`] + /// is used to create an [`Input::Live`]. + Lazy( + /// A trait object which can be used to (re)create a usable byte stream. + Box, + ), + /// An initialised byte source. + /// + /// This contains a raw byte stream, the lazy initialiser that was used, + /// as well as any symphonia-specific format data and/or hints. + Live( + /// The byte source, plus symphonia-specific data. + LiveInput, + /// The struct used to initialise this source, if available. + /// + /// This is used to recreate the stream when a source does not support + /// backward seeking, if present. + Option>, + ), } impl Input { - /// Creates a floating-point PCM Input from a given reader. - pub fn float_pcm(is_stereo: bool, reader: Reader) -> Input { - Input { - metadata: Default::default(), - stereo: is_stereo, - reader, - kind: Codec::FloatPcm, - container: Container::Raw, - pos: 0, - } - } - - /// Creates a new Input using (at least) the given reader, codec, and container. - pub fn new( - stereo: bool, - reader: Reader, - kind: Codec, - container: Container, - metadata: Option, - ) -> Self { - Input { - metadata: metadata.unwrap_or_default().into(), - stereo, - reader, - kind, - container, - pos: 0, - } - } - - /// Returns whether the inner [`Reader`] implements [`Seek`]. + /// Requests auxiliary metadata which can be accessed without parsing the file. /// - /// [`Reader`]: reader::Reader + /// This method will never be called by songbird but allows, for instance, access to metadata + /// which might only be visible to a web crawler, e.g., uploader or source URL. + /// + /// This requires that the [`Input`] has a [`Compose`] available to use, otherwise it + /// will always fail with [`AudioStreamError::Unsupported`]. + pub async fn aux_metadata(&mut self) -> Result { + match self { + Self::Lazy(ref mut composer) => composer.aux_metadata().await.map_err(Into::into), + Self::Live(_, Some(ref mut composer)) => + composer.aux_metadata().await.map_err(Into::into), + Self::Live(_, None) => Err(AuxMetadataError::NoCompose), + } + } + + /// Tries to get any information about this audio stream acquired during parsing. + /// + /// Only exists when this input is both [`Self::Live`] and has been fully parsed. + /// In general, you probably want to use [`Self::aux_metadata`]. + pub fn metadata(&mut self) -> Result { + if let Self::Live(live, _) = self { + live.metadata() + } else { + Err(MetadataError::NotLive) + } + } + + /// Initialises (but does not parse) an [`Input::Lazy`] into an [`Input::Live`], + /// placing blocking I/O on the current thread. + /// + /// This requires a [`TokioHandle`] to a tokio runtime to spawn any `async` sources. + /// + /// *This is a blocking operation. If you wish to use this from an async task, you + /// must do so via [`Self::make_live_async`].* + /// + /// This is a no-op for an [`Input::Live`]. + pub fn make_live(self, handle: &TokioHandle) -> Result { + if let Self::Lazy(mut lazy) = self { + let (created, lazy) = if lazy.should_create_async() { + let (tx, rx) = flume::bounded(1); + handle.spawn(async move { + let out = lazy.create_async().await; + drop(tx.send_async((out, lazy))); + }); + rx.recv().map_err(|_| { + let err_msg: Box = + "async Input create handler panicked".into(); + AudioStreamError::Fail(err_msg) + })? + } else { + (lazy.create(), lazy) + }; + + Ok(Self::Live(LiveInput::Raw(created?), Some(lazy))) + } else { + Ok(self) + } + } + + /// Initialises (but does not parse) an [`Input::Lazy`] into an [`Input::Live`], + /// placing blocking I/O on the a `spawn_blocking` executor. + /// + /// This is a no-op for an [`Input::Live`]. + pub async fn make_live_async(self) -> Result { + if let Self::Lazy(mut lazy) = self { + let (created, lazy) = if lazy.should_create_async() { + (lazy.create_async().await, lazy) + } else { + tokio::task::spawn_blocking(move || (lazy.create(), lazy)) + .await + .map_err(|_| { + let err_msg: Box = + "synchronous Input create handler panicked".into(); + AudioStreamError::Fail(err_msg) + })? + }; + + Ok(Self::Live(LiveInput::Raw(created?), Some(lazy))) + } else { + Ok(self) + } + } + + /// Initialises and parses an [`Input::Lazy`] into an [`Input::Live`], + /// placing blocking I/O on the current thread. + /// + /// This requires a [`TokioHandle`] to a tokio runtime to spawn any `async` sources. + /// If you can't access one, then consider manually using [`LiveInput::promote`]. + /// + /// *This is a blocking operation. Symphonia uses standard library I/O (e.g., [`Read`], [`Seek`]). + /// If you wish to use this from an async task, you must do so within `spawn_blocking`.* + /// + /// [`Read`]: https://doc.rust-lang.org/std/io/trait.Read.html /// [`Seek`]: https://doc.rust-lang.org/std/io/trait.Seek.html - pub fn is_seekable(&self) -> bool { - self.reader.is_seekable() - } - - /// Returns whether the read audio signal is stereo (or mono). - pub fn is_stereo(&self) -> bool { - self.stereo - } - - /// Returns the type of the inner [`Codec`]. - /// - /// [`Codec`]: Codec - pub fn get_type(&self) -> CodecType { - (&self.kind).into() - } - - /// Mixes the output of this stream into a 20ms stereo audio buffer. - #[inline] - pub fn mix(&mut self, float_buffer: &mut [f32; STEREO_FRAME_SIZE], volume: f32) -> usize { - self.add_float_pcm_frame(float_buffer, self.stereo, volume) - .unwrap_or(0) - } - - /// Seeks the stream to the given time, if possible. - /// - /// Returns the actual time reached. - pub fn seek_time(&mut self, time: Duration) -> Option { - let future_pos = utils::timestamp_to_byte_count(time, self.stereo); - Seek::seek(self, SeekFrom::Start(future_pos as u64)) - .ok() - .map(|a| utils::byte_count_to_timestamp(a as usize, self.stereo)) - } - - fn read_inner(&mut self, buffer: &mut [u8], ignore_decode: bool) -> IoResult { - // This implementation of Read converts the input stream - // to floating point output. - let sample_len = mem::size_of::(); - let float_space = buffer.len() / sample_len; - let mut written_floats = 0; - - // TODO: better decouple codec and container here. - // this is a little bit backwards, and assumes the bottom cases are always raw... - let out = match &mut self.kind { - Codec::Opus(decoder_state) => { - if matches!(self.container, Container::Raw) { - return Err(IoError::new( - IoErrorKind::InvalidInput, - "Raw container cannot demarcate Opus frames.", - )); - } - - if ignore_decode { - // If we're less than one frame away from the end of cheap seeking, - // then we must decode to make sure the next starting offset is correct. - - // Step one: use up the remainder of the frame. - let mut aud_skipped = - decoder_state.current_frame.len() - decoder_state.frame_pos; - - decoder_state.frame_pos = 0; - decoder_state.current_frame.truncate(0); - - // Step two: take frames if we can. - while buffer.len() - aud_skipped >= STEREO_FRAME_BYTE_SIZE { - decoder_state.should_reset = true; - - let frame = self - .container - .next_frame_length(&mut self.reader, CodecType::Opus)?; - self.reader.consume(frame.frame_len); - - aud_skipped += STEREO_FRAME_BYTE_SIZE; - } - - Ok(aud_skipped) - } else { - // get new frame *if needed* - if decoder_state.frame_pos == decoder_state.current_frame.len() { - let mut decoder = decoder_state.decoder.lock(); - - if decoder_state.should_reset { - decoder - .reset_state() - .expect("Critical failure resetting decoder."); - decoder_state.should_reset = false; - } - let frame = self - .container - .next_frame_length(&mut self.reader, CodecType::Opus)?; - - let mut opus_data_buffer = [0u8; 4000]; - - decoder_state - .current_frame - .resize(decoder_state.current_frame.capacity(), 0.0); - - let seen = - Read::read(&mut self.reader, &mut opus_data_buffer[..frame.frame_len])?; - - let samples = decoder - .decode_float( - Some((&opus_data_buffer[..seen]).try_into().unwrap()), - (&mut decoder_state.current_frame[..]).try_into().unwrap(), - false, - ) - .unwrap_or(0); - - decoder_state.current_frame.truncate(2 * samples); - decoder_state.frame_pos = 0; - } - - // read from frame which is present. - let mut buffer = buffer; - - let start = decoder_state.frame_pos; - let to_write = float_space.min(decoder_state.current_frame.len() - start); - for val in &decoder_state.current_frame[start..start + float_space] { - buffer.write_f32::(*val)?; - } - decoder_state.frame_pos += to_write; - written_floats = to_write; - - Ok(written_floats * mem::size_of::()) - } + pub fn make_playable( + self, + codecs: &CodecRegistry, + probe: &Probe, + handle: &TokioHandle, + ) -> Result { + let out = self.make_live(handle)?; + match out { + Self::Lazy(_) => unreachable!(), + Self::Live(input, lazy) => { + let promoted = input.promote(codecs, probe)?; + Ok(Self::Live(promoted, lazy)) }, - Codec::Pcm => { - let mut buffer = buffer; - while written_floats < float_space { - if let Ok(signal) = self.reader.read_i16::() { - buffer.write_f32::(f32::from(signal) / 32768.0)?; - written_floats += 1; - } else { - break; - } - } - Ok(written_floats * mem::size_of::()) - }, - Codec::FloatPcm => Read::read(&mut self.reader, buffer), - }; - - out.map(|v| { - self.pos += v; - v - }) - } - - fn cheap_consume(&mut self, count: usize) -> IoResult { - let mut scratch = [0u8; STEREO_FRAME_BYTE_SIZE * 4]; - let len = scratch.len(); - let mut done = 0; - - loop { - let read = self.read_inner(&mut scratch[..len.min(count - done)], true)?; - if read == 0 { - break; - } - done += read; - } - - Ok(done) - } - - pub(crate) fn supports_passthrough(&self) -> bool { - match &self.kind { - Codec::Opus(state) => state.allow_passthrough, - _ => false, } } - pub(crate) fn read_opus_frame(&mut self, buffer: &mut [u8]) -> IoResult { - // Called in event of opus passthrough. - if let Codec::Opus(state) = &mut self.kind { - // step 1: align to frame. - self.pos += state.current_frame.len() - state.frame_pos; + /// Initialises and parses an [`Input::Lazy`] into an [`Input::Live`], + /// placing blocking I/O on a tokio blocking thread. + pub async fn make_playable_async( + self, + codecs: &'static CodecRegistry, + probe: &'static Probe, + ) -> Result { + let out = self.make_live_async().await?; + match out { + Self::Lazy(_) => unreachable!(), + Self::Live(input, lazy) => { + let promoted = tokio::task::spawn_blocking(move || input.promote(codecs, probe)) + .await + .map_err(|_| MakePlayableError::Panicked)??; + Ok(Self::Live(promoted, lazy)) + }, + } + } - state.frame_pos = 0; - state.current_frame.truncate(0); - - // step 2: read new header. - let frame = self - .container - .next_frame_length(&mut self.reader, CodecType::Opus)?; - - // step 3: read in bytes. - self.reader - .read_exact(&mut buffer[..frame.frame_len]) - .map(|_| { - self.pos += STEREO_FRAME_BYTE_SIZE; - frame.frame_len - }) + /// Returns whether this audio stream is full initialised, parsed, and + /// ready to play (e.g., `Self::Live(LiveInput::Parsed(p), _)`). + #[must_use] + pub fn is_playable(&self) -> bool { + if let Self::Live(input, _) = self { + input.is_playable() } else { - Err(IoError::new( - IoErrorKind::InvalidInput, - "Frame passthrough not supported for this file.", - )) + false } } - pub(crate) fn prep_with_handle(&mut self, handle: Handle) { - self.reader.prep_with_handle(handle); - } -} - -impl Read for Input { - fn read(&mut self, buffer: &mut [u8]) -> IoResult { - self.read_inner(buffer, false) - } -} - -impl Seek for Input { - fn seek(&mut self, pos: SeekFrom) -> IoResult { - let mut target = self.pos; - match pos { - SeekFrom::Start(pos) => { - target = pos as usize; - }, - SeekFrom::Current(rel) => { - target = target.wrapping_add(rel as usize); - }, - SeekFrom::End(_pos) => unimplemented!(), - } - - debug!("Seeking to {:?}", pos); - - (if target == self.pos { - Ok(0) - } else if let Some(conversion) = self.container.try_seek_trivial(self.get_type()) { - let inside_target = (target * conversion) / mem::size_of::(); - Seek::seek(&mut self.reader, SeekFrom::Start(inside_target as u64)).map(|inner_dest| { - let outer_dest = ((inner_dest as usize) * mem::size_of::()) / conversion; - self.pos = outer_dest; - outer_dest - }) - } else if target > self.pos { - // seek in the next amount, disabling decoding if need be. - let shift = target - self.pos; - self.cheap_consume(shift) + /// Returns a reference to the live input, if it has been created via + /// [`Self::make_live`] or [`Self::make_live_async`]. + #[must_use] + pub fn live(&self) -> Option<&LiveInput> { + if let Self::Live(input, _) = self { + Some(input) } else { - // start from scratch, then seek in... - Seek::seek( - &mut self.reader, - SeekFrom::Start(self.container.input_start() as u64), - )?; - - self.cheap_consume(target) - }) - .map(|_| self.pos as u64) + None + } } -} -/// Extension trait to pull frames of audio from a byte source. -pub(crate) trait ReadAudioExt { - fn add_float_pcm_frame( - &mut self, - float_buffer: &mut [f32; STEREO_FRAME_SIZE], - true_stereo: bool, - volume: f32, - ) -> Option; - - fn consume(&mut self, amt: usize) -> usize - where - Self: Sized; -} - -impl ReadAudioExt for R { - fn add_float_pcm_frame( - &mut self, - float_buffer: &mut [f32; STEREO_FRAME_SIZE], - stereo: bool, - volume: f32, - ) -> Option { - // IDEA: Read in 8 floats at a time, then use iterator code - // to gently nudge the compiler into vectorising for us. - // Max SIMD float32 lanes is 8 on AVX, older archs use a divisor of this - // e.g., 4. - const SAMPLE_LEN: usize = mem::size_of::(); - const FLOAT_COUNT: usize = 512; - let mut simd_float_bytes = [0u8; FLOAT_COUNT * SAMPLE_LEN]; - let mut simd_float_buf = [0f32; FLOAT_COUNT]; - - let mut frame_pos = 0; - - // Code duplication here is because unifying these codepaths - // with a dynamic chunk size is not zero-cost. - if stereo { - let mut max_bytes = STEREO_FRAME_BYTE_SIZE; - - while frame_pos < float_buffer.len() { - let progress = self - .read(&mut simd_float_bytes[..max_bytes.min(FLOAT_COUNT * SAMPLE_LEN)]) - .and_then(|byte_len| { - let target = byte_len / SAMPLE_LEN; - (&simd_float_bytes[..byte_len]) - .read_f32_into::(&mut simd_float_buf[..target]) - .map(|_| target) - }) - .map(|f32_len| { - let new_pos = frame_pos + f32_len; - for (el, new_el) in float_buffer[frame_pos..new_pos] - .iter_mut() - .zip(&simd_float_buf[..f32_len]) - { - *el += volume * new_el; - } - (new_pos, f32_len) - }); - - match progress { - Ok((new_pos, delta)) => { - frame_pos = new_pos; - max_bytes -= delta * SAMPLE_LEN; - - if delta == 0 { - break; - } - }, - Err(ref e) => - return if e.kind() == IoErrorKind::UnexpectedEof { - error!("EOF unexpectedly: {:?}", e); - Some(frame_pos) - } else { - error!("Input died unexpectedly: {:?}", e); - None - }, - } - } + /// Returns a mutable reference to the live input, if it been created via + /// [`Self::make_live`] or [`Self::make_live_async`]. + pub fn live_mut(&mut self) -> Option<&mut LiveInput> { + if let Self::Live(ref mut input, _) = self { + Some(input) } else { - let mut max_bytes = MONO_FRAME_BYTE_SIZE; - - while frame_pos < float_buffer.len() { - let progress = self - .read(&mut simd_float_bytes[..max_bytes.min(FLOAT_COUNT * SAMPLE_LEN)]) - .and_then(|byte_len| { - let target = byte_len / SAMPLE_LEN; - (&simd_float_bytes[..byte_len]) - .read_f32_into::(&mut simd_float_buf[..target]) - .map(|_| target) - }) - .map(|f32_len| { - let new_pos = frame_pos + (2 * f32_len); - for (els, new_el) in float_buffer[frame_pos..new_pos] - .chunks_exact_mut(2) - .zip(&simd_float_buf[..f32_len]) - { - let sample = volume * new_el; - els[0] += sample; - els[1] += sample; - } - (new_pos, f32_len) - }); - - match progress { - Ok((new_pos, delta)) => { - frame_pos = new_pos; - max_bytes -= delta * SAMPLE_LEN; - - if delta == 0 { - break; - } - }, - Err(ref e) => - return if e.kind() == IoErrorKind::UnexpectedEof { - Some(frame_pos) - } else { - error!("Input died unexpectedly: {:?}", e); - None - }, - } - } + None } - - Some(frame_pos * SAMPLE_LEN) } - fn consume(&mut self, amt: usize) -> usize { - io::copy(&mut self.by_ref().take(amt as u64), &mut io::sink()).unwrap_or(0) as usize + /// Returns a reference to the data parsed from this input stream, if it has + /// been made available via [`Self::make_playable`] or [`LiveInput::promote`]. + #[must_use] + pub fn parsed(&self) -> Option<&Parsed> { + self.live().and_then(LiveInput::parsed) + } + + /// Returns a mutable reference to the data parsed from this input stream, if it + /// has been made available via [`Self::make_playable`] or [`LiveInput::promote`]. + pub fn parsed_mut(&mut self) -> Option<&mut Parsed> { + self.live_mut().and_then(LiveInput::parsed_mut) } } -#[cfg(test)] -mod tests { - use super::*; - use crate::test_utils::*; +impl + Send + Sync + 'static> From for Input { + fn from(val: T) -> Self { + let raw_src = LiveInput::Raw(AudioStream { + input: Box::new(Cursor::new(val)), + hint: None, + }); - #[test] - fn float_pcm_input_unchanged_mono() { - let data = make_sine(50 * MONO_FRAME_SIZE, false); - let mut input = Input::new( - false, - data.clone().into(), - Codec::FloatPcm, - Container::Raw, - None, - ); - - let mut out_vec = vec![]; - - let len = input.read_to_end(&mut out_vec).unwrap(); - assert_eq!(out_vec[..len], data[..]); - } - - #[test] - fn float_pcm_input_unchanged_stereo() { - let data = make_sine(50 * MONO_FRAME_SIZE, true); - let mut input = Input::new( - true, - data.clone().into(), - Codec::FloatPcm, - Container::Raw, - None, - ); - - let mut out_vec = vec![]; - - let len = input.read_to_end(&mut out_vec).unwrap(); - assert_eq!(out_vec[..len], data[..]); - } - - #[test] - fn pcm_input_becomes_float_mono() { - let data = make_pcm_sine(50 * MONO_FRAME_SIZE, false); - let mut input = Input::new(false, data.clone().into(), Codec::Pcm, Container::Raw, None); - - let mut out_vec = vec![]; - let _len = input.read_to_end(&mut out_vec).unwrap(); - - let mut i16_window = &data[..]; - let mut float_window = &out_vec[..]; - - while i16_window.len() != 0 { - let before = i16_window.read_i16::().unwrap() as f32; - let after = float_window.read_f32::().unwrap(); - - let diff = (before / 32768.0) - after; - - assert!(diff.abs() < f32::EPSILON); - } - } - - #[test] - fn pcm_input_becomes_float_stereo() { - let data = make_pcm_sine(50 * MONO_FRAME_SIZE, true); - let mut input = Input::new(true, data.clone().into(), Codec::Pcm, Container::Raw, None); - - let mut out_vec = vec![]; - let _len = input.read_to_end(&mut out_vec).unwrap(); - - let mut i16_window = &data[..]; - let mut float_window = &out_vec[..]; - - while i16_window.len() != 0 { - let before = i16_window.read_i16::().unwrap() as f32; - let after = float_window.read_f32::().unwrap(); - - let diff = (before / 32768.0) - after; - - assert!(diff.abs() < f32::EPSILON); - } + Input::Live(raw_src, None) } } diff --git a/src/input/parsed.rs b/src/input/parsed.rs new file mode 100644 index 0000000..6ae1627 --- /dev/null +++ b/src/input/parsed.rs @@ -0,0 +1,31 @@ +use symphonia_core::{codecs::Decoder, formats::FormatReader, probe::ProbedMetadata}; + +/// An audio file which has had its headers parsed and decoder state built. +pub struct Parsed { + /// Audio packet, seeking, and state access for all tracks in a file. + /// + /// This may be used to access packets one at a time from the input file. + /// Additionally, this exposes container-level and per track metadata which + /// have been extracted. + pub format: Box, + + /// Decoder state for the chosen track. + pub decoder: Box, + + /// The chosen track's ID. + /// + /// This is required to identify the correct packet stream inside the container. + pub track_id: u32, + + /// Metadata extracted by symphonia while detecting a file's format. + /// + /// Typically, this detects metadata *outside* the file's core format (i.e., + /// ID3 tags in MP3 and WAV files). + pub meta: ProbedMetadata, + + /// Whether the contained format supports arbitrary seeking. + /// + /// If set to false, Songbird will attempt to recreate the input if + /// it must seek backwards. + pub supports_backseek: bool, +} diff --git a/src/input/reader.rs b/src/input/reader.rs deleted file mode 100644 index 98e0055..0000000 --- a/src/input/reader.rs +++ /dev/null @@ -1,152 +0,0 @@ -//! Raw handlers for input bytestreams. - -use super::*; -use std::{ - fmt::{Debug, Error as FormatError, Formatter}, - fs::File, - io::{ - BufReader, - Cursor, - Error as IoError, - ErrorKind as IoErrorKind, - Read, - Result as IoResult, - Seek, - SeekFrom, - }, - result::Result as StdResult, -}; -use streamcatcher::{Catcher, TxCatcher}; -pub use symphonia_core::io::MediaSource; - -/// Usable data/byte sources for an audio stream. -/// -/// Users may define their own data sources using [`Extension`]. -/// -/// [`Extension`]: Reader::Extension -pub enum Reader { - /// Piped output of another program (i.e., [`ffmpeg`]). - /// - /// Does not support seeking. - /// - /// [`ffmpeg`]: super::ffmpeg - Pipe(BufReader), - /// A cached, raw in-memory store, provided by Songbird. - /// - /// Supports seeking. - Memory(Catcher>), - /// A cached, Opus-compressed in-memory store, provided by Songbird. - /// - /// Supports seeking. - Compressed(TxCatcher, OpusCompressor>), - /// A source which supports seeking by recreating its inout stream. - /// - /// Supports seeking. - Restartable(Restartable), - /// A basic user-provided source. - /// - /// Seeking support depends on underlying `MediaSource` implementation. - Extension(Box), -} - -impl Reader { - /// Returns whether the given source implements [`Seek`]. - /// - /// This might be an expensive operation and might involve blocking IO. In such cases, it is - /// advised to cache the return value when possible. - /// - /// [`Seek`]: https://doc.rust-lang.org/std/io/trait.Seek.html - pub fn is_seekable(&self) -> bool { - use Reader::*; - match self { - Restartable(_) | Compressed(_) | Memory(_) => true, - Extension(source) => source.is_seekable(), - _ => false, - } - } - - /// A source contained in a local file. - pub fn from_file(file: File) -> Self { - Self::Extension(Box::new(file)) - } - - /// A source contained as an array in memory. - pub fn from_memory(buf: Vec) -> Self { - Self::Extension(Box::new(Cursor::new(buf))) - } - - #[allow(clippy::single_match)] - pub(crate) fn prep_with_handle(&mut self, handle: Handle) { - use Reader::*; - match self { - Restartable(r) => r.prep_with_handle(handle), - _ => {}, - } - } - - #[allow(clippy::single_match)] - pub(crate) fn make_playable(&mut self) { - use Reader::*; - match self { - Restartable(r) => r.make_playable(), - _ => {}, - } - } -} - -impl Read for Reader { - fn read(&mut self, buffer: &mut [u8]) -> IoResult { - use Reader::*; - match self { - Pipe(a) => Read::read(a, buffer), - Memory(a) => Read::read(a, buffer), - Compressed(a) => Read::read(a, buffer), - Restartable(a) => Read::read(a, buffer), - Extension(a) => a.read(buffer), - } - } -} - -impl Seek for Reader { - fn seek(&mut self, pos: SeekFrom) -> IoResult { - use Reader::*; - match self { - Pipe(_) => Err(IoError::new( - IoErrorKind::InvalidInput, - "Seeking not supported on Reader of this type.", - )), - Memory(a) => Seek::seek(a, pos), - Compressed(a) => Seek::seek(a, pos), - Restartable(a) => Seek::seek(a, pos), - Extension(a) => - if a.is_seekable() { - a.seek(pos) - } else { - Err(IoError::new( - IoErrorKind::InvalidInput, - "Seeking not supported on Reader of this type.", - )) - }, - } - } -} - -impl Debug for Reader { - fn fmt(&self, f: &mut Formatter<'_>) -> StdResult<(), FormatError> { - use Reader::*; - let field = match self { - Pipe(a) => format!("{:?}", a), - Memory(a) => format!("{:?}", a), - Compressed(a) => format!("{:?}", a), - Restartable(a) => format!("{:?}", a), - Extension(_) => "Extension".to_string(), - }; - f.debug_tuple("Reader").field(&field).finish() - } -} - -impl From> for Reader { - fn from(val: Vec) -> Self { - Self::from_memory(val) - } -} diff --git a/src/input/restartable.rs b/src/input/restartable.rs deleted file mode 100644 index 353affb..0000000 --- a/src/input/restartable.rs +++ /dev/null @@ -1,455 +0,0 @@ -//! A source which supports seeking by recreating its input stream. -//! -//! This is intended for use with single-use audio tracks which -//! may require looping or seeking, but where additional memory -//! cannot be spared. Forward seeks will drain the track until reaching -//! the desired timestamp. -//! -//! Restarting occurs by temporarily pausing the track, running the restart -//! mechanism, and then passing the handle back to the mixer thread. Until -//! success/failure is confirmed, the track produces silence. - -use super::*; -use async_trait::async_trait; -use flume::{Receiver, TryRecvError}; -use std::{ - ffi::OsStr, - fmt::{Debug, Error as FormatError, Formatter}, - io::{Error as IoError, ErrorKind as IoErrorKind, Read, Result as IoResult, Seek, SeekFrom}, - result::Result as StdResult, - time::Duration, -}; - -type Recreator = Box; -type RecreateChannel = Receiver, Recreator)>>; - -// Use options here to make "take" more doable from a mut ref. -enum LazyProgress { - Dead(Box, Option, Codec, Container), - Live(Box, Option), - Working(Codec, Container, bool, RecreateChannel), -} - -impl Debug for LazyProgress { - fn fmt(&self, f: &mut Formatter<'_>) -> StdResult<(), FormatError> { - match self { - LazyProgress::Dead(meta, _, codec, container) => f - .debug_tuple("Dead") - .field(meta) - .field(&"") - .field(codec) - .field(container) - .finish(), - LazyProgress::Live(input, _) => - f.debug_tuple("Live").field(input).field(&"").finish(), - LazyProgress::Working(codec, container, stereo, chan) => f - .debug_tuple("Working") - .field(codec) - .field(container) - .field(stereo) - .field(chan) - .finish(), - } - } -} - -/// A wrapper around a method to create a new [`Input`] which -/// seeks backward by recreating the source. -/// -/// The main purpose of this wrapper is to enable seeking on -/// incompatible sources (i.e., ffmpeg output) and to ease resource -/// consumption for commonly reused/shared tracks. [`Compressed`] -/// and [`Memory`] offer the same functionality with different -/// tradeoffs. -/// -/// This is intended for use with single-use audio tracks which -/// may require looping or seeking, but where additional memory -/// cannot be spared. Forward seeks will drain the track until reaching -/// the desired timestamp. -/// -/// [`Input`]: Input -/// [`Memory`]: cached::Memory -/// [`Compressed`]: cached::Compressed -#[derive(Debug)] -pub struct Restartable { - async_handle: Option, - position: usize, - source: LazyProgress, -} - -impl Restartable { - /// Create a new source, which can be restarted using a `recreator` function. - /// - /// Lazy sources will not run their input recreator until the first byte - /// is needed, or are sent [`Track::make_playable`]/[`TrackHandle::make_playable`]. - /// - /// [`Track::make_playable`]: crate::tracks::Track::make_playable - /// [`TrackHandle::make_playable`]: crate::tracks::TrackHandle::make_playable - pub async fn new(mut recreator: impl Restart + Send + 'static, lazy: bool) -> Result { - if lazy { - recreator - .lazy_init() - .await - .map(move |(meta, kind, codec)| Self { - async_handle: None, - position: 0, - source: LazyProgress::Dead( - meta.unwrap_or_default().into(), - Some(Box::new(recreator)), - kind, - codec, - ), - }) - } else { - recreator.call_restart(None).await.map(move |source| Self { - async_handle: None, - position: 0, - source: LazyProgress::Live(source.into(), Some(Box::new(recreator))), - }) - } - } - - /// Create a new restartable ffmpeg source for a local file. - pub async fn ffmpeg + Send + Clone + Sync + 'static>( - path: P, - lazy: bool, - ) -> Result { - Self::new(FfmpegRestarter { path }, lazy).await - } - - /// Create a new restartable ytdl source. - /// - /// The cost of restarting and seeking will probably be *very* high: - /// expect a pause if you seek backwards. - pub async fn ytdl + Send + Clone + Sync + 'static>( - uri: P, - lazy: bool, - ) -> Result { - Self::new(YtdlRestarter { uri }, lazy).await - } - - /// Create a new restartable ytdl source, using the first result of a youtube search. - /// - /// The cost of restarting and seeking will probably be *very* high: - /// expect a pause if you seek backwards. - pub async fn ytdl_search(name: impl AsRef, lazy: bool) -> Result { - Self::ytdl(format!("ytsearch1:{}", name.as_ref()), lazy).await - } - - pub(crate) fn prep_with_handle(&mut self, handle: Handle) { - self.async_handle = Some(handle); - } - - pub(crate) fn make_playable(&mut self) { - if matches!(self.source, LazyProgress::Dead(_, _, _, _)) { - // This read triggers creation of a source, and is guaranteed not to modify any internals. - // It will harmlessly write out zeroes into the target buffer. - let mut bytes = [0u8; 0]; - let _ = Read::read(self, &mut bytes[..]); - } - } -} - -/// Trait used to create an instance of a [`Reader`] at instantiation and when -/// a backwards seek is needed. -/// -/// [`Reader`]: reader::Reader -#[async_trait] -pub trait Restart { - /// Tries to create a replacement source. - async fn call_restart(&mut self, time: Option) -> Result; - - /// Optionally retrieve metadata for a source which has been lazily initialised. - /// - /// This is particularly useful for sources intended to be queued, which - /// should occupy few resources when not live BUT have as much information as - /// possible made available at creation. - async fn lazy_init(&mut self) -> Result<(Option, Codec, Container)>; -} - -struct FfmpegRestarter

-where - P: AsRef + Send + Sync, -{ - path: P, -} - -#[async_trait] -impl

Restart for FfmpegRestarter

-where - P: AsRef + Send + Sync, -{ - async fn call_restart(&mut self, time: Option) -> Result { - if let Some(time) = time { - let is_stereo = is_stereo(self.path.as_ref()) - .await - .unwrap_or_else(|_e| (false, Default::default())); - let stereo_val = if is_stereo.0 { "2" } else { "1" }; - - let ts = format!("{:.3}", time.as_secs_f64()); - _ffmpeg_optioned( - self.path.as_ref(), - &["-ss", &ts], - &[ - "-f", - "s16le", - "-ac", - stereo_val, - "-ar", - "48000", - "-acodec", - "pcm_f32le", - "-", - ], - Some(is_stereo), - ) - .await - } else { - ffmpeg(self.path.as_ref()).await - } - } - - async fn lazy_init(&mut self) -> Result<(Option, Codec, Container)> { - is_stereo(self.path.as_ref()) - .await - .map(|(_stereo, metadata)| (Some(metadata), Codec::FloatPcm, Container::Raw)) - } -} - -struct YtdlRestarter

-where - P: AsRef + Send + Sync, -{ - uri: P, -} - -#[async_trait] -impl

Restart for YtdlRestarter

-where - P: AsRef + Send + Sync, -{ - async fn call_restart(&mut self, time: Option) -> Result { - if let Some(time) = time { - let ts = format!("{:.3}", time.as_secs_f64()); - - _ytdl(self.uri.as_ref(), &["-ss", &ts]).await - } else { - ytdl(self.uri.as_ref()).await - } - } - - async fn lazy_init(&mut self) -> Result<(Option, Codec, Container)> { - _ytdl_metadata(self.uri.as_ref()) - .await - .map(|m| (Some(m), Codec::FloatPcm, Container::Raw)) - } -} - -impl From for Input { - fn from(mut src: Restartable) -> Self { - let (meta, stereo, kind, container) = match &mut src.source { - LazyProgress::Dead(ref mut m, _rec, kind, container) => { - let stereo = m.channels == Some(2); - (Some(m.take()), stereo, kind.clone(), *container) - }, - LazyProgress::Live(ref mut input, _rec) => ( - Some(input.metadata.take()), - input.stereo, - input.kind.clone(), - input.container, - ), - // This branch should never be taken: this is an emergency measure. - LazyProgress::Working(kind, container, stereo, _) => - (None, *stereo, kind.clone(), *container), - }; - Input::new(stereo, Reader::Restartable(src), kind, container, meta) - } -} - -// How do these work at a high level? -// If you need to restart, send a request to do this to the async context. -// if a request is pending, then just output all zeroes. - -impl Read for Restartable { - fn read(&mut self, buffer: &mut [u8]) -> IoResult { - use LazyProgress::*; - let (out_val, march_pos, next_source) = match &mut self.source { - Dead(meta, rec, kind, container) => { - let stereo = meta.channels == Some(2); - let handle = self.async_handle.clone(); - let new_chan = if let Some(rec) = rec.take() { - Some(regenerate_channel( - rec, - 0, - stereo, - kind.clone(), - *container, - handle, - )?) - } else { - return Err(IoError::new( - IoErrorKind::UnexpectedEof, - "Illegal state: taken recreator was observed.".to_string(), - )); - }; - - // Then, output all zeroes. - for el in buffer.iter_mut() { - *el = 0; - } - (Ok(buffer.len()), false, new_chan) - }, - Live(source, _) => (Read::read(source, buffer), true, None), - Working(_, _, _, chan) => { - match chan.try_recv() { - Ok(Ok((mut new_source, recreator))) => { - // Completed! - // Do read, then replace inner progress. - let bytes_read = Read::read(&mut new_source, buffer); - - (bytes_read, true, Some(Live(new_source, Some(recreator)))) - }, - Ok(Err(source_error)) => { - let e = Err(IoError::new( - IoErrorKind::UnexpectedEof, - format!("Failed to create new reader: {:?}.", source_error), - )); - (e, false, None) - }, - Err(TryRecvError::Empty) => { - // Output all zeroes. - for el in buffer.iter_mut() { - *el = 0; - } - (Ok(buffer.len()), false, None) - }, - Err(_) => { - let e = Err(IoError::new( - IoErrorKind::UnexpectedEof, - "Failed to create new reader: dropped.", - )); - (e, false, None) - }, - } - }, - }; - - if let Some(src) = next_source { - self.source = src; - } - - if march_pos { - out_val.map(|a| { - self.position += a; - a - }) - } else { - out_val - } - } -} - -impl Seek for Restartable { - fn seek(&mut self, pos: SeekFrom) -> IoResult { - let _local_pos = self.position as u64; - - use SeekFrom::*; - match pos { - Start(offset) => { - let offset = offset as usize; - let handle = self.async_handle.clone(); - - use LazyProgress::*; - match &mut self.source { - Dead(meta, rec, kind, container) => { - // regen at given start point - self.source = if let Some(rec) = rec.take() { - regenerate_channel( - rec, - offset, - meta.channels == Some(2), - kind.clone(), - *container, - handle, - )? - } else { - return Err(IoError::new( - IoErrorKind::UnexpectedEof, - "Illegal state: taken recreator was observed.".to_string(), - )); - }; - - self.position = offset; - }, - Live(input, rec) => { - if offset < self.position { - // regen at given start point - // We're going back in time. - self.source = if let Some(rec) = rec.take() { - regenerate_channel( - rec, - offset, - input.stereo, - input.kind.clone(), - input.container, - handle, - )? - } else { - return Err(IoError::new( - IoErrorKind::UnexpectedEof, - "Illegal state: taken recreator was observed.".to_string(), - )); - }; - - self.position = offset; - } else { - // march on with live source. - self.position += input.consume(offset - self.position); - } - }, - Working(_, _, _, _) => { - return Err(IoError::new( - IoErrorKind::Interrupted, - "Previous seek in progress.", - )); - }, - } - - Ok(offset as u64) - }, - End(_offset) => Err(IoError::new( - IoErrorKind::InvalidInput, - "End point for Restartables is not known.", - )), - Current(_offset) => unimplemented!(), - } - } -} - -fn regenerate_channel( - mut rec: Recreator, - offset: usize, - stereo: bool, - kind: Codec, - container: Container, - handle: Option, -) -> IoResult { - if let Some(handle) = handle.as_ref() { - let (tx, rx) = flume::bounded(1); - - handle.spawn(async move { - let ret_val = rec - .call_restart(Some(utils::byte_count_to_timestamp(offset, stereo))) - .await; - - let _ = tx.send(ret_val.map(Box::new).map(|v| (v, rec))); - }); - - Ok(LazyProgress::Working(kind, container, stereo, rx)) - } else { - Err(IoError::new( - IoErrorKind::Interrupted, - "Cannot safely call seek until provided an async context handle.", - )) - } -} diff --git a/src/input/sources/file.rs b/src/input/sources/file.rs new file mode 100644 index 0000000..756a816 --- /dev/null +++ b/src/input/sources/file.rs @@ -0,0 +1,80 @@ +use crate::input::{AudioStream, AudioStreamError, AuxMetadata, Compose, Input}; +use std::{error::Error, ffi::OsStr, path::Path}; +use symphonia_core::{io::MediaSource, probe::Hint}; +use tokio::process::Command; + +/// A lazily instantiated local file. +#[derive(Clone, Debug)] +pub struct File> { + path: P, +} + +impl> File

{ + /// Creates a lazy file object, which will open the target path. + /// + /// This is infallible as the path is only checked during creation. + pub fn new(path: P) -> Self { + Self { path } + } +} + +impl + Send + Sync + 'static> From> for Input { + fn from(val: File

) -> Self { + Input::Lazy(Box::new(val)) + } +} + +#[async_trait::async_trait] +impl + Send + Sync> Compose for File

{ + fn create(&mut self) -> Result>, AudioStreamError> { + let err: Box = + "Files should be created asynchronously.".to_string().into(); + Err(AudioStreamError::Fail(err)) + } + + async fn create_async( + &mut self, + ) -> Result>, AudioStreamError> { + let file = tokio::fs::File::open(&self.path) + .await + .map_err(|io| AudioStreamError::Fail(Box::new(io)))?; + + let input = Box::new(file.into_std().await); + + let mut hint = Hint::default(); + if let Some(ext) = self.path.as_ref().extension().and_then(OsStr::to_str) { + hint.with_extension(ext); + } + + Ok(AudioStream { + input, + hint: Some(hint), + }) + } + + fn should_create_async(&self) -> bool { + true + } + + /// Probes for metadata about this audio files using `ffprobe`. + async fn aux_metadata(&mut self) -> Result { + let args = [ + "-v", + "quiet", + "-of", + "json", + "-show_format", + "-show_streams", + "-i", + ]; + + let output = Command::new("ffprobe") + .args(&args) + .output() + .await + .map_err(|e| AudioStreamError::Fail(Box::new(e)))?; + + AuxMetadata::from_ffprobe_json(&output.stdout[..]) + .map_err(|e| AudioStreamError::Fail(Box::new(e))) + } +} diff --git a/src/input/sources/http.rs b/src/input/sources/http.rs new file mode 100644 index 0000000..f956129 --- /dev/null +++ b/src/input/sources/http.rs @@ -0,0 +1,292 @@ +use crate::input::{ + AsyncAdapterStream, + AsyncMediaSource, + AudioStream, + AudioStreamError, + Compose, + Input, +}; +use async_trait::async_trait; +use futures::TryStreamExt; +use pin_project::pin_project; +use reqwest::{ + header::{HeaderMap, ACCEPT_RANGES, CONTENT_LENGTH, CONTENT_TYPE, RANGE, RETRY_AFTER}, + Client, +}; +use std::{ + io::{Error as IoError, ErrorKind as IoErrorKind, Result as IoResult, SeekFrom}, + pin::Pin, + task::{Context, Poll}, + time::Duration, +}; +use symphonia_core::{io::MediaSource, probe::Hint}; +use tokio::io::{AsyncRead, AsyncSeek, ReadBuf}; +use tokio_util::io::StreamReader; + +/// A lazily instantiated HTTP request. +#[derive(Clone, Debug)] +pub struct HttpRequest { + /// A reqwest client instance used to send the HTTP GET request. + pub client: Client, + /// The target URL of the required resource. + pub request: String, + /// HTTP header fields to add to any created requests. + pub headers: HeaderMap, + /// Content length, used as an upper bound in range requests if known. + /// + /// This is only needed for certain domains who expect to see a value like + /// `range: bytes=0-1023` instead of the simpler `range: bytes=0-` (such as + /// Youtube). + pub content_length: Option, +} + +impl HttpRequest { + #[must_use] + /// Create a lazy HTTP request. + pub fn new(client: Client, request: String) -> Self { + Self::new_with_headers(client, request, HeaderMap::default()) + } + + #[must_use] + /// Create a lazy HTTP request. + pub fn new_with_headers(client: Client, request: String, headers: HeaderMap) -> Self { + HttpRequest { + client, + request, + headers, + content_length: None, + } + } + + async fn create_stream( + &mut self, + offset: Option, + ) -> Result<(HttpStream, Option), AudioStreamError> { + let mut resp = self.client.get(&self.request).headers(self.headers.clone()); + + match (offset, self.content_length) { + (Some(offset), None) => { + resp = resp.header(RANGE, format!("bytes={}-", offset)); + }, + (offset, Some(max)) => { + resp = resp.header( + RANGE, + format!("bytes={}-{}", offset.unwrap_or(0), max.saturating_sub(1)), + ); + }, + _ => {}, + } + + let resp = resp + .send() + .await + .map_err(|e| AudioStreamError::Fail(Box::new(e)))?; + + if let Some(t) = resp.headers().get(RETRY_AFTER) { + t.to_str() + .map_err(|_| { + let msg: Box = + "Retry-after field contained non-ASCII data.".into(); + AudioStreamError::Fail(msg) + }) + .and_then(|str_text| { + str_text.parse().map_err(|_| { + let msg: Box = + "Retry-after field was non-numeric.".into(); + AudioStreamError::Fail(msg) + }) + }) + .and_then(|t| Err(AudioStreamError::RetryIn(Duration::from_secs(t)))) + } else { + let headers = resp.headers(); + + let hint = headers + .get(CONTENT_TYPE) + .and_then(|val| val.to_str().ok()) + .map(|val| { + let mut out = Hint::default(); + out.mime_type(val); + out + }); + + let len = headers + .get(CONTENT_LENGTH) + .and_then(|val| val.to_str().ok()) + .and_then(|val| val.parse().ok()); + + let resume = headers + .get(ACCEPT_RANGES) + .and_then(|a| a.to_str().ok()) + .and_then(|a| { + if a == "bytes" { + Some(self.clone()) + } else { + None + } + }); + + let stream = Box::new(StreamReader::new( + resp.bytes_stream() + .map_err(|e| IoError::new(IoErrorKind::Other, e)), + )); + + let input = HttpStream { + stream, + len, + resume, + }; + + Ok((input, hint)) + } + } +} + +#[pin_project] +struct HttpStream { + #[pin] + stream: Box, + len: Option, + resume: Option, +} + +impl AsyncRead for HttpStream { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + AsyncRead::poll_read(self.project().stream, cx, buf) + } +} + +impl AsyncSeek for HttpStream { + fn start_seek(self: Pin<&mut Self>, _position: SeekFrom) -> IoResult<()> { + Err(IoErrorKind::Unsupported.into()) + } + + fn poll_complete(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + unreachable!() + } +} + +#[async_trait] +impl AsyncMediaSource for HttpStream { + fn is_seekable(&self) -> bool { + false + } + + async fn byte_len(&self) -> Option { + self.len + } + + async fn try_resume( + &mut self, + offset: u64, + ) -> Result, AudioStreamError> { + if let Some(resume) = &mut self.resume { + resume + .create_stream(Some(offset)) + .await + .map(|a| Box::new(a.0) as Box) + } else { + Err(AudioStreamError::Unsupported) + } + } +} + +#[async_trait] +impl Compose for HttpRequest { + fn create(&mut self) -> Result>, AudioStreamError> { + Err(AudioStreamError::Unsupported) + } + + async fn create_async( + &mut self, + ) -> Result>, AudioStreamError> { + self.create_stream(None).await.map(|(input, hint)| { + let stream = AsyncAdapterStream::new(Box::new(input), 64 * 1024); + + AudioStream { + input: Box::new(stream) as Box, + hint, + } + }) + } + + fn should_create_async(&self) -> bool { + true + } +} + +impl From for Input { + fn from(val: HttpRequest) -> Self { + Input::Lazy(Box::new(val)) + } +} + +#[cfg(test)] +mod tests { + use reqwest::Client; + + use super::*; + use crate::{ + constants::test_data::{HTTP_OPUS_TARGET, HTTP_TARGET, HTTP_WEBM_TARGET}, + input::input_tests::*, + }; + + #[tokio::test] + #[ntest::timeout(10_000)] + async fn http_track_plays() { + track_plays_mixed(|| HttpRequest::new(Client::new(), HTTP_TARGET.into())).await; + } + + #[tokio::test] + #[ntest::timeout(10_000)] + async fn http_forward_seek_correct() { + forward_seek_correct(|| HttpRequest::new(Client::new(), HTTP_TARGET.into())).await; + } + + #[tokio::test] + #[ntest::timeout(10_000)] + async fn http_backward_seek_correct() { + backward_seek_correct(|| HttpRequest::new(Client::new(), HTTP_TARGET.into())).await; + } + + // NOTE: this covers youtube audio in a non-copyright-violating way, since + // those depend on an HttpRequest internally anyhow. + #[tokio::test] + #[ntest::timeout(10_000)] + async fn http_opus_track_plays() { + track_plays_passthrough(|| HttpRequest::new(Client::new(), HTTP_OPUS_TARGET.into())).await; + } + + #[tokio::test] + #[ntest::timeout(10_000)] + async fn http_opus_forward_seek_correct() { + forward_seek_correct(|| HttpRequest::new(Client::new(), HTTP_OPUS_TARGET.into())).await; + } + + #[tokio::test] + #[ntest::timeout(10_000)] + async fn http_opus_backward_seek_correct() { + backward_seek_correct(|| HttpRequest::new(Client::new(), HTTP_OPUS_TARGET.into())).await; + } + + #[tokio::test] + #[ntest::timeout(10_000)] + async fn http_webm_track_plays() { + track_plays_passthrough(|| HttpRequest::new(Client::new(), HTTP_WEBM_TARGET.into())).await; + } + + #[tokio::test] + #[ntest::timeout(10_000)] + async fn http_webm_forward_seek_correct() { + forward_seek_correct(|| HttpRequest::new(Client::new(), HTTP_WEBM_TARGET.into())).await; + } + + #[tokio::test] + #[ntest::timeout(10_000)] + async fn http_webm_backward_seek_correct() { + backward_seek_correct(|| HttpRequest::new(Client::new(), HTTP_WEBM_TARGET.into())).await; + } +} diff --git a/src/input/sources/mod.rs b/src/input/sources/mod.rs new file mode 100644 index 0000000..47ce9cc --- /dev/null +++ b/src/input/sources/mod.rs @@ -0,0 +1,5 @@ +mod file; +mod http; +mod ytdl; + +pub use self::{file::*, http::*, ytdl::*}; diff --git a/src/input/sources/ytdl.rs b/src/input/sources/ytdl.rs new file mode 100644 index 0000000..c796235 --- /dev/null +++ b/src/input/sources/ytdl.rs @@ -0,0 +1,158 @@ +use crate::input::{ + metadata::ytdl::Output, + AudioStream, + AudioStreamError, + AuxMetadata, + Compose, + HttpRequest, + Input, +}; +use async_trait::async_trait; +use reqwest::{ + header::{HeaderMap, HeaderName, HeaderValue}, + Client, +}; +use std::error::Error; +use symphonia_core::io::MediaSource; +use tokio::process::Command; + +const YOUTUBE_DL_COMMAND: &str = "yt-dlp"; + +/// A lazily instantiated call to download a file, finding its URL via youtube-dl. +/// +/// By default, this uses yt-dlp and is backed by an [`HttpRequest`]. This handler +/// attempts to find the best audio-only source (typically `WebM`, enabling low-cost +/// Opus frame passthrough). +/// +/// [`HttpRequest`]: super::HttpRequest +#[derive(Clone, Debug)] +pub struct YoutubeDl { + program: &'static str, + client: Client, + metadata: Option, + url: String, +} + +impl YoutubeDl { + /// Creates a lazy request to select an audio stream from `url`, using "yt-dlp". + /// + /// This requires a reqwest client: ideally, one should be created and shared between + /// all requests. + #[must_use] + pub fn new(client: Client, url: String) -> Self { + Self::new_ytdl_like(YOUTUBE_DL_COMMAND, client, url) + } + + /// Creates a lazy request to select an audio stream from `url` as in [`new`], using `program`. + /// + /// [`new`]: Self::new + #[must_use] + pub fn new_ytdl_like(program: &'static str, client: Client, url: String) -> Self { + Self { + program, + client, + metadata: None, + url, + } + } + + async fn query(&mut self) -> Result { + let ytdl_args = ["-j", &self.url, "-f", "ba[abr>0][vcodec=none]/best"]; + + let output = Command::new(self.program) + .args(&ytdl_args) + .output() + .await + .map_err(|e| AudioStreamError::Fail(Box::new(e)))?; + + let stdout: Output = serde_json::from_slice(&output.stdout[..]) + .map_err(|e| AudioStreamError::Fail(Box::new(e)))?; + + self.metadata = Some(stdout.as_aux_metadata()); + + Ok(stdout) + } +} + +impl From for Input { + fn from(val: YoutubeDl) -> Self { + Input::Lazy(Box::new(val)) + } +} + +#[async_trait] +impl Compose for YoutubeDl { + fn create(&mut self) -> Result>, AudioStreamError> { + Err(AudioStreamError::Unsupported) + } + + async fn create_async( + &mut self, + ) -> Result>, AudioStreamError> { + let stdout = self.query().await?; + + let mut headers = HeaderMap::default(); + + if let Some(map) = stdout.http_headers { + headers.extend(map.iter().filter_map(|(k, v)| { + Some(( + HeaderName::from_bytes(k.as_bytes()).ok()?, + HeaderValue::from_str(v).ok()?, + )) + })); + } + + let mut req = HttpRequest { + client: self.client.clone(), + request: stdout.url, + headers, + content_length: stdout.filesize, + }; + + req.create_async().await + } + + fn should_create_async(&self) -> bool { + true + } + + async fn aux_metadata(&mut self) -> Result { + if let Some(meta) = self.metadata.as_ref() { + return Ok(meta.clone()); + } + + self.query().await?; + + self.metadata.clone().ok_or_else(|| { + let msg: Box = + "Failed to instansiate any metadata... Should be unreachable.".into(); + AudioStreamError::Fail(msg) + }) + } +} + +#[cfg(test)] +mod tests { + use reqwest::Client; + + use super::*; + use crate::{constants::test_data::YTDL_TARGET, input::input_tests::*}; + + #[tokio::test] + #[ntest::timeout(20_000)] + async fn ytdl_track_plays() { + track_plays_mixed(|| YoutubeDl::new(Client::new(), YTDL_TARGET.into())).await; + } + + #[tokio::test] + #[ntest::timeout(20_000)] + async fn ytdl_forward_seek_correct() { + forward_seek_correct(|| YoutubeDl::new(Client::new(), YTDL_TARGET.into())).await; + } + + #[tokio::test] + #[ntest::timeout(20_000)] + async fn ytdl_backward_seek_correct() { + backward_seek_correct(|| YoutubeDl::new(Client::new(), YTDL_TARGET.into())).await; + } +} diff --git a/src/input/utils.rs b/src/input/utils.rs index d6072da..282545f 100644 --- a/src/input/utils.rs +++ b/src/input/utils.rs @@ -4,26 +4,30 @@ use crate::constants::*; use audiopus::{coder::Decoder, Channels, Result as OpusResult, SampleRate}; use std::{mem, time::Duration}; -/// Calculates the sample position in a FloatPCM stream from a timestamp. +/// Calculates the sample position in a `FloatPCM` stream from a timestamp. +#[must_use] pub fn timestamp_to_sample_count(timestamp: Duration, stereo: bool) -> usize { ((timestamp.as_millis() as usize) * (MONO_FRAME_SIZE / FRAME_LEN_MS)) << stereo as usize } -/// Calculates the time position in a FloatPCM stream from a sample index. +/// Calculates the time position in a `FloatPCM` stream from a sample index. +#[must_use] pub fn sample_count_to_timestamp(amt: usize, stereo: bool) -> Duration { Duration::from_millis((((amt * FRAME_LEN_MS) / MONO_FRAME_SIZE) as u64) >> stereo as u64) } -/// Calculates the byte position in a FloatPCM stream from a timestamp. +/// Calculates the byte position in a `FloatPCM` stream from a timestamp. /// /// Each sample is sized by `mem::size_of::() == 4usize`. +#[must_use] pub fn timestamp_to_byte_count(timestamp: Duration, stereo: bool) -> usize { timestamp_to_sample_count(timestamp, stereo) * mem::size_of::() } -/// Calculates the time position in a FloatPCM stream from a byte index. +/// Calculates the time position in a `FloatPCM` stream from a byte index. /// /// Each sample is sized by `mem::size_of::() == 4usize`. +#[must_use] pub fn byte_count_to_timestamp(amt: usize, stereo: bool) -> Duration { sample_count_to_timestamp(amt / mem::size_of::(), stereo) } diff --git a/src/input/ytdl_src.rs b/src/input/ytdl_src.rs deleted file mode 100644 index e929780..0000000 --- a/src/input/ytdl_src.rs +++ /dev/null @@ -1,176 +0,0 @@ -use super::{ - children_to_reader, - error::{Error, Result}, - Codec, - Container, - Input, - Metadata, -}; -use serde_json::Value; -use std::{ - io::{BufRead, BufReader, Read}, - process::{Command, Stdio}, -}; -use tokio::{process::Command as TokioCommand, task}; -use tracing::trace; - -const YOUTUBE_DL_COMMAND: &str = if cfg!(feature = "youtube-dlc") { - "youtube-dlc" -} else if cfg!(feature = "yt-dlp") { - "yt-dlp" -} else { - "youtube-dl" -}; - -/// Creates a streamed audio source with `youtube-dl` and `ffmpeg`. -/// -/// This source is not seek-compatible. -/// If you need looping or track seeking, then consider using -/// [`Restartable::ytdl`]. -/// -/// `youtube-dlc` and `yt-dlp` are also useable by enabling the `youtube-dlc` -/// and `yt-dlp` features respectively. -/// -/// [`Restartable::ytdl`]: crate::input::restartable::Restartable::ytdl -pub async fn ytdl(uri: impl AsRef) -> Result { - _ytdl(uri.as_ref(), &[]).await -} - -pub(crate) async fn _ytdl(uri: &str, pre_args: &[&str]) -> Result { - let ytdl_args = [ - "--print-json", - "-f", - "webm[abr>0]/bestaudio/best", - "-R", - "infinite", - "--no-playlist", - "--ignore-config", - "--no-warnings", - uri, - "-o", - "-", - ]; - - let ffmpeg_args = [ - "-f", - "s16le", - "-ac", - "2", - "-ar", - "48000", - "-acodec", - "pcm_f32le", - "-", - ]; - - let mut youtube_dl = Command::new(YOUTUBE_DL_COMMAND) - .args(&ytdl_args) - .stdin(Stdio::null()) - .stderr(Stdio::piped()) - .stdout(Stdio::piped()) - .spawn()?; - - // This rigmarole is required due to the inner synchronous reading context. - let stderr = youtube_dl.stderr.take(); - let (returned_stderr, value) = task::spawn_blocking(move || { - let mut s = stderr.unwrap(); - let out: Result = { - let mut o_vec = vec![]; - let mut serde_read = BufReader::new(s.by_ref()); - // Newline... - if let Ok(len) = serde_read.read_until(0xA, &mut o_vec) { - serde_json::from_slice(&o_vec[..len]).map_err(|err| Error::Json { - error: err, - parsed_text: std::str::from_utf8(&o_vec).unwrap_or_default().to_string(), - }) - } else { - Result::Err(Error::Metadata) - } - }; - - (s, out) - }) - .await - .map_err(|_| Error::Metadata)?; - - youtube_dl.stderr = Some(returned_stderr); - - let taken_stdout = youtube_dl.stdout.take().ok_or(Error::Stdout)?; - - let ffmpeg = Command::new("ffmpeg") - .args(pre_args) - .arg("-i") - .arg("-") - .args(&ffmpeg_args) - .stdin(taken_stdout) - .stderr(Stdio::null()) - .stdout(Stdio::piped()) - .spawn()?; - - let metadata = Metadata::from_ytdl_output(value?); - - trace!("ytdl metadata {:?}", metadata); - - Ok(Input::new( - true, - children_to_reader::(vec![youtube_dl, ffmpeg]), - Codec::FloatPcm, - Container::Raw, - Some(metadata), - )) -} - -pub(crate) async fn _ytdl_metadata(uri: &str) -> Result { - // Most of these flags are likely unused, but we want identical search - // and/or selection as the above functions. - let ytdl_args = [ - "-j", - "-f", - "webm[abr>0]/bestaudio/best", - "-R", - "infinite", - "--no-playlist", - "--ignore-config", - "--no-warnings", - uri, - "-o", - "-", - ]; - - let youtube_dl_output = TokioCommand::new(YOUTUBE_DL_COMMAND) - .args(&ytdl_args) - .stdin(Stdio::null()) - .output() - .await?; - - let o_vec = youtube_dl_output.stderr; - - let end = (&o_vec) - .iter() - .position(|el| *el == 0xA) - .unwrap_or_else(|| o_vec.len()); - - let value = serde_json::from_slice(&o_vec[..end]).map_err(|err| Error::Json { - error: err, - parsed_text: std::str::from_utf8(&o_vec).unwrap_or_default().to_string(), - })?; - - let metadata = Metadata::from_ytdl_output(value); - - Ok(metadata) -} - -/// Creates a streamed audio source from YouTube search results with `youtube-dl(c)`,`ffmpeg`, and `ytsearch`. -/// Takes the first video listed from the YouTube search. -/// -/// This source is not seek-compatible. -/// If you need looping or track seeking, then consider using -/// [`Restartable::ytdl_search`]. -/// -/// `youtube-dlc` and `yt-dlp` are also useable by enabling the `youtube-dlc` -/// and `yt-dlp` features respectively. -/// -/// [`Restartable::ytdl_search`]: crate::input::restartable::Restartable::ytdl_search -pub async fn ytdl_search(name: impl AsRef) -> Result { - ytdl(&format!("ytsearch1:{}", name.as_ref())).await -} diff --git a/src/join.rs b/src/join.rs index 76a4e93..a721821 100644 --- a/src/join.rs +++ b/src/join.rs @@ -1,6 +1,6 @@ //! Future types for gateway interactions. -#[cfg(feature = "driver-core")] +#[cfg(feature = "driver")] use crate::error::ConnectionResult; use crate::{ error::{JoinError, JoinResult}, @@ -18,7 +18,7 @@ use flume::r#async::RecvFut; use pin_project::pin_project; use tokio::time::{self, Timeout}; -#[cfg(feature = "driver-core")] +#[cfg(feature = "driver")] /// Future for a call to [`Call::join`]. /// /// This future `await`s Discord's response *and* @@ -40,7 +40,7 @@ pub struct Join { state: JoinState, } -#[cfg(feature = "driver-core")] +#[cfg(feature = "driver")] impl Join { pub(crate) fn new( driver: RecvFut<'static, ConnectionResult<()>>, @@ -55,7 +55,7 @@ impl Join { } } -#[cfg(feature = "driver-core")] +#[cfg(feature = "driver")] impl Future for Join { type Output = JoinResult<()>; @@ -96,7 +96,7 @@ impl Future for Join { } } -#[cfg(feature = "driver-core")] +#[cfg(feature = "driver")] #[derive(Copy, Clone, Eq, PartialEq)] enum JoinState { BeforeGw, @@ -137,6 +137,7 @@ impl Future for JoinGateway { } } +#[allow(clippy::large_enum_variant)] #[pin_project(project = JoinClassProj)] enum JoinClass { WithTimeout(#[pin] Timeout>), diff --git a/src/lib.rs b/src/lib.rs index d685d7e..50fc45e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,7 +3,7 @@ html_favicon_url = "https://raw.githubusercontent.com/serenity-rs/songbird/current/songbird-ico.png" )] #![deny(missing_docs)] -#![deny(broken_intra_doc_links)] +#![deny(rustdoc::broken_intra_doc_links)] //! ![project logo][logo] //! //! Songbird is an async, cross-library compatible voice system for Discord, written in Rust. @@ -16,7 +16,7 @@ //! can run the songbird voice driver. //! * And, by default, a fully featured voice system featuring events, queues, RT(C)P packet //! handling, seeking on compatible streams, shared multithreaded audio stream caches, -//! and direct Opus data passthrough from DCA files. +//! and direct Opus data passthrough. //! //! ## Intents //! Songbird's gateway functionality requires you to specify the `GUILD_VOICE_STATES` intent. @@ -25,6 +25,26 @@ //! Full examples showing various types of functionality and integrations can be found //! in [this crate's examples directory]. //! +//! ## Codec support +//! Songbird supports all [codecs and formats provided by Symphonia] (pure-Rust), with Opus support +//! provided by [audiopus] (an FFI wrapper for libopus). +//! +//! **By default, *Songbird will not request any codecs from Symphonia*.** To change this, in your own +//! project you will need to depend on Symphonia as well. +//! +//! ```toml +//! ## Including songbird alone gives you support for Opus via the DCA file format. +//! [dependencies.songbird] +//! version = "0.4" +//! features = ["builtin-queue"] +//! +//! ## To get additional codecs, you *must* add Symphonia yourself. +//! ## This includes the default formats (MKV/WebM, Ogg, Wave) and codecs (FLAC, PCM, Vorbis)... +//! [dependencies.symphonia] +//! version = "0.5" +//! features = ["aac", "mp3", "isomp4", "alac"] # ...as well as any extras you need! +//! ``` +//! //! ## Attribution //! //! Songbird's logo is based upon the copyright-free image ["Black-Capped Chickadee"] by George Gorgas White. @@ -36,55 +56,64 @@ //! ["Black-Capped Chickadee"]: https://www.oldbookillustrations.com/illustrations/black-capped-chickadee/ //! [`ConnectionInfo`]: struct@ConnectionInfo //! [lavalink]: https://github.com/freyacodes/Lavalink +//! [codecs and formats provided by Symphonia]: https://github.com/pdeljanov/Symphonia#formats-demuxers +//! [audiopus]: https://github.com/lakelezz/audiopus -#[cfg(all(feature = "youtube-dlc", feature = "yt-dlp"))] -compile_error!("feature \"youtube-dlc\" and feature \"yt-dlp\" cannot be enabled at the same time"); +#![warn(clippy::pedantic)] +#![allow( + // Allowed as they are too pedantic + clippy::module_name_repetitions, + clippy::wildcard_imports, + clippy::too_many_lines, + clippy::cast_lossless, + clippy::cast_sign_loss, + clippy::cast_possible_wrap, + clippy::cast_precision_loss, + clippy::cast_possible_truncation, + // TODO: would require significant rewriting of all existing docs + clippy::missing_errors_doc, +)] mod config; pub mod constants; -#[cfg(feature = "driver-core")] +#[cfg(feature = "driver")] pub mod driver; pub mod error; -#[cfg(feature = "driver-core")] +#[cfg(feature = "driver")] pub mod events; -#[cfg(feature = "gateway-core")] +#[cfg(feature = "gateway")] mod handler; pub mod id; pub(crate) mod info; -#[cfg(feature = "driver-core")] +#[cfg(feature = "driver")] pub mod input; -#[cfg(feature = "gateway-core")] +#[cfg(feature = "gateway")] pub mod join; -#[cfg(feature = "gateway-core")] +#[cfg(feature = "gateway")] mod manager; #[cfg(feature = "serenity")] pub mod serenity; -#[cfg(feature = "gateway-core")] +#[cfg(feature = "gateway")] pub mod shards; -#[cfg(feature = "driver-core")] +#[cfg(feature = "driver")] pub mod tracks; -#[cfg(feature = "driver-core")] +#[cfg(feature = "driver")] mod ws; -#[cfg(feature = "driver-core")] +#[cfg(feature = "driver")] pub use discortp as packet; -#[cfg(feature = "driver-core")] +#[cfg(feature = "driver")] pub use serenity_voice_model as model; -#[cfg(feature = "driver-core")] +#[cfg(feature = "driver")] pub use typemap_rev as typemap; -#[cfg(test)] -use utils as test_utils; - -#[cfg(feature = "driver-core")] +#[cfg(feature = "driver")] pub use crate::{ driver::Driver, events::{CoreEvent, Event, EventContext, EventHandler, TrackEvent}, - input::{ffmpeg, ytdl}, - tracks::create_player, }; -#[cfg(feature = "gateway-core")] +#[cfg(feature = "gateway")] pub use crate::{handler::*, manager::*}; #[cfg(feature = "serenity")] diff --git a/src/manager.rs b/src/manager.rs index 6ebad0e..ef81145 100644 --- a/src/manager.rs +++ b/src/manager.rs @@ -1,3 +1,5 @@ +#[cfg(feature = "serenity")] +use crate::shards::SerenitySharder; use crate::{ error::{JoinError, JoinResult}, id::{ChannelId, GuildId, UserId}, @@ -11,6 +13,7 @@ use async_trait::async_trait; use dashmap::DashMap; #[cfg(feature = "serenity")] use futures::channel::mpsc::UnboundedSender as Sender; +use once_cell::sync::OnceCell; use parking_lot::RwLock as PRwLock; #[cfg(feature = "serenity")] use serenity::{ @@ -29,10 +32,9 @@ use twilight_gateway::Cluster; #[cfg(feature = "twilight")] use twilight_model::gateway::event::Event as TwilightEvent; -#[derive(Clone, Copy, Debug, Default)] +#[derive(Clone, Copy, Debug)] struct ClientData { shard_count: u64, - initialised: bool, user_id: UserId, } @@ -44,7 +46,7 @@ struct ClientData { /// [`Call`]: Call #[derive(Debug)] pub struct Songbird { - client_data: PRwLock, + client_data: OnceCell, calls: DashMap>>, sharder: Sharder, config: PRwLock>, @@ -57,8 +59,9 @@ impl Songbird { /// This must be [registered] after creation. /// /// [registered]: crate::serenity::register_with + #[must_use] pub fn serenity() -> Arc { - Self::serenity_from_config(Default::default()) + Self::serenity_from_config(Config::default()) } #[cfg(feature = "serenity")] @@ -67,11 +70,12 @@ impl Songbird { /// This must be [registered] after creation. /// /// [registered]: crate::serenity::register_with + #[must_use] pub fn serenity_from_config(config: Config) -> Arc { Arc::new(Self { - client_data: Default::default(), - calls: Default::default(), - sharder: Sharder::Serenity(Default::default()), + client_data: OnceCell::new(), + calls: DashMap::new(), + sharder: Sharder::Serenity(SerenitySharder::default()), config: Some(config).into(), }) } @@ -88,7 +92,7 @@ impl Songbird { where U: Into, { - Self::twilight_from_config(cluster, user_id, Default::default()) + Self::twilight_from_config(cluster, user_id, Config::default()) } #[cfg(feature = "twilight")] @@ -104,12 +108,11 @@ impl Songbird { U: Into, { Self { - client_data: PRwLock::new(ClientData { + client_data: OnceCell::with_value(ClientData { shard_count: cluster.config().shard_scheme().total(), - initialised: true, user_id: user_id.into(), }), - calls: Default::default(), + calls: DashMap::new(), sharder: Sharder::TwilightCluster(cluster), config: Some(config).into(), } @@ -122,15 +125,12 @@ impl Songbird { /// /// [`::twilight`]: #method.twilight pub fn initialise_client_data>(&self, shard_count: u64, user_id: U) { - let mut client_data = self.client_data.write(); - - if client_data.initialised { - return; - } - - client_data.shard_count = shard_count; - client_data.user_id = user_id.into(); - client_data.initialised = true; + self.client_data + .set(ClientData { + shard_count, + user_id: user_id.into(), + }) + .ok(); } /// Retrieves a [`Call`] for the given guild, if one already exists. @@ -161,8 +161,12 @@ impl Songbird { self.calls .entry(guild_id) .or_insert_with(|| { - let info = self.manager_info(); - let shard = shard_id(guild_id.0, info.shard_count); + let info = self + .client_data + .get() + .expect("Manager has not been initialised"); + + let shard = shard_id(guild_id.0.get(), info.shard_count); let shard_handle = self .sharder .get_shard(shard) @@ -192,13 +196,7 @@ impl Songbird { *config = Some(new_config); } - fn manager_info(&self) -> ClientData { - let client_data = self.client_data.write(); - - *client_data - } - - #[cfg(feature = "driver-core")] + #[cfg(feature = "driver")] /// Connects to a target by retrieving its relevant [`Call`] and /// connecting, or creating the handler if required. /// @@ -230,7 +228,7 @@ impl Songbird { self._join(guild_id.into(), channel_id.into()).await } - #[cfg(feature = "driver-core")] + #[cfg(feature = "driver")] async fn _join( &self, guild_id: GuildId, @@ -371,7 +369,11 @@ impl Songbird { } }, TwilightEvent::VoiceStateUpdate(v) => { - if v.0.user_id.get() != self.client_data.read().user_id.0 { + if self + .client_data + .get() + .map_or(true, |data| v.0.user_id.into_nonzero() != data.user_id.0) + { return; } @@ -390,16 +392,16 @@ impl Songbird { #[cfg(feature = "serenity")] #[async_trait] impl VoiceGatewayManager for Songbird { - async fn initialise(&self, shard_count: u64, user_id: SerenityUser) { + async fn initialise(&self, shard_count: u32, user_id: SerenityUser) { debug!( "Initialising Songbird for Serenity: ID {:?}, {} Shards", user_id, shard_count ); - self.initialise_client_data(shard_count, user_id); + self.initialise_client_data(shard_count as u64, user_id); debug!("Songbird ({:?}) Initialised!", user_id); } - async fn register_shard(&self, shard_id: u64, sender: Sender) { + async fn register_shard(&self, shard_id: u32, sender: Sender) { debug!( "Registering Serenity shard handle {} with Songbird", shard_id @@ -408,7 +410,7 @@ impl VoiceGatewayManager for Songbird { debug!("Registered shard handle {}.", shard_id); } - async fn deregister_shard(&self, shard_id: u64) { + async fn deregister_shard(&self, shard_id: u32) { debug!( "Deregistering Serenity shard handle {} with Songbird", shard_id @@ -427,7 +429,11 @@ impl VoiceGatewayManager for Songbird { } async fn state_update(&self, guild_id: SerenityGuild, voice_state: &VoiceState) { - if voice_state.user_id.0 != self.client_data.read().user_id.0 { + if self + .client_data + .get() + .map_or(true, |data| voice_state.user_id.0 != data.user_id.0) + { return; } diff --git a/src/serenity.rs b/src/serenity.rs index 92d66e1..eb9872d 100644 --- a/src/serenity.rs +++ b/src/serenity.rs @@ -11,9 +11,10 @@ use serenity::{ use std::sync::Arc; /// Zero-size type used to retrieve the registered [`Songbird`] instance -/// from serenity's inner TypeMap. +/// from serenity's inner [`TypeMap`]. /// /// [`Songbird`]: Songbird +/// [`TypeMap`]: serenity::prelude::TypeMap pub struct SongbirdKey; impl TypeMapKey for SongbirdKey { @@ -63,10 +64,13 @@ pub trait SerenityInit { /// access via [`get`]. /// /// [`get`]: get + #[must_use] fn register_songbird(self) -> Self; /// Registers a given Songbird voice system with serenity, as above. + #[must_use] fn register_songbird_with(self, voice: Arc) -> Self; /// Registers a Songbird voice system serenity, based on the given configuration. + #[must_use] fn register_songbird_from_config(self, config: Config) -> Self; } diff --git a/src/shards.rs b/src/shards.rs index 92613ab..9185b23 100644 --- a/src/shards.rs +++ b/src/shards.rs @@ -1,10 +1,9 @@ //! Handlers for sending packets over sharded connections. -use crate::{ - error::{JoinError, JoinResult}, - id::*, -}; +use crate::{error::JoinResult, id::*}; use async_trait::async_trait; +#[cfg(feature = "serenity")] +use dashmap::DashMap; use derivative::Derivative; #[cfg(feature = "serenity")] use futures::channel::mpsc::{TrySendError, UnboundedSender as Sender}; @@ -14,8 +13,8 @@ use serde_json::json; #[cfg(feature = "serenity")] use serenity::gateway::InterMessage; #[cfg(feature = "serenity")] -use std::{collections::HashMap, result::Result as StdResult}; -use std::{num::NonZeroU64, sync::Arc}; +use std::result::Result as StdResult; +use std::sync::Arc; use tracing::{debug, error}; #[cfg(feature = "twilight")] use twilight_gateway::{Cluster, Shard as TwilightShard}; @@ -52,10 +51,13 @@ pub trait GenericSharder { impl Sharder { /// Returns a new handle to the required inner shard. + #[allow(clippy::must_use_candidate)] // get_or_insert_shard_handle has side effects pub fn get_shard(&self, shard_id: u64) -> Option { match self { #[cfg(feature = "serenity")] - Sharder::Serenity(s) => Some(Shard::Serenity(s.get_or_insert_shard_handle(shard_id))), + Sharder::Serenity(s) => Some(Shard::Serenity( + s.get_or_insert_shard_handle(shard_id as u32), + )), #[cfg(feature = "twilight")] Sharder::TwilightCluster(t) => Some(Shard::TwilightCluster(t.clone(), shard_id)), #[cfg(feature = "twilight")] @@ -68,18 +70,20 @@ impl Sharder { #[cfg(feature = "serenity")] impl Sharder { #[allow(unreachable_patterns)] - pub(crate) fn register_shard_handle(&self, shard_id: u64, sender: Sender) { - match self { - Sharder::Serenity(s) => s.register_shard_handle(shard_id, sender), - _ => error!("Called serenity management function on a non-serenity Songbird instance."), + pub(crate) fn register_shard_handle(&self, shard_id: u32, sender: Sender) { + if let Sharder::Serenity(s) = self { + s.register_shard_handle(shard_id, sender); + } else { + error!("Called serenity management function on a non-serenity Songbird instance."); } } #[allow(unreachable_patterns)] - pub(crate) fn deregister_shard_handle(&self, shard_id: u64) { - match self { - Sharder::Serenity(s) => s.deregister_shard_handle(shard_id), - _ => error!("Called serenity management function on a non-serenity Songbird instance."), + pub(crate) fn deregister_shard_handle(&self, shard_id: u32) { + if let Sharder::Serenity(s) = self { + s.deregister_shard_handle(shard_id); + } else { + error!("Called serenity management function on a non-serenity Songbird instance."); } } } @@ -90,29 +94,22 @@ impl Sharder { /// /// This is updated and maintained by the library, and is designed to prevent /// message loss during rebalances and reconnects. -pub struct SerenitySharder(PRwLock>>); +pub struct SerenitySharder(DashMap>); #[cfg(feature = "serenity")] impl SerenitySharder { - fn get_or_insert_shard_handle(&self, shard_id: u64) -> Arc { - ({ - let map_read = self.0.read(); - map_read.get(&shard_id).cloned() - }) - .unwrap_or_else(|| { - let mut map_read = self.0.write(); - map_read.entry(shard_id).or_default().clone() - }) + fn get_or_insert_shard_handle(&self, shard_id: u32) -> Arc { + self.0.entry(shard_id).or_default().clone() } - fn register_shard_handle(&self, shard_id: u64, sender: Sender) { + fn register_shard_handle(&self, shard_id: u32, sender: Sender) { // Write locks are only used to add new entries to the map. let handle = self.get_or_insert_shard_handle(shard_id); handle.register(sender); } - fn deregister_shard_handle(&self, shard_id: u64) { + fn deregister_shard_handle(&self, shard_id: u32) { // Write locks are only used to add new entries to the map. let handle = self.get_or_insert_shard_handle(shard_id); @@ -120,7 +117,7 @@ impl SerenitySharder { } } -#[derive(Derivative)] +#[derive(Derivative, Clone)] #[derivative(Debug)] #[non_exhaustive] /// A reference to an individual websocket connection. @@ -138,22 +135,6 @@ pub enum Shard { Generic(#[derivative(Debug = "ignore")] Arc), } -impl Clone for Shard { - fn clone(&self) -> Self { - use Shard::*; - - match self { - #[cfg(feature = "serenity")] - Serenity(handle) => Serenity(Arc::clone(handle)), - #[cfg(feature = "twilight")] - TwilightCluster(handle, id) => TwilightCluster(Arc::clone(handle), *id), - #[cfg(feature = "twilight")] - TwilightShard(handle) => TwilightShard(Arc::clone(handle)), - Generic(handle) => Generic(Arc::clone(handle)), - } - } -} - #[async_trait] impl VoiceUpdate for Shard { async fn update_voice_state( @@ -163,13 +144,6 @@ impl VoiceUpdate for Shard { self_deaf: bool, self_mute: bool, ) -> JoinResult<()> { - let nz_guild_id = NonZeroU64::new(guild_id.0).ok_or(JoinError::IllegalGuild)?; - - let nz_channel_id = match channel_id { - Some(c) => Some(NonZeroU64::new(c.0).ok_or(JoinError::IllegalChannel)?), - None => None, - }; - match self { #[cfg(feature = "serenity")] Shard::Serenity(handle) => { @@ -183,20 +157,20 @@ impl VoiceUpdate for Shard { } }); - handle.send(InterMessage::Json(map))?; + handle.send(InterMessage::json(map.to_string()))?; Ok(()) }, #[cfg(feature = "twilight")] Shard::TwilightCluster(handle, shard_id) => { - let channel_id = nz_channel_id.map(From::from); - let cmd = TwilightVoiceState::new(nz_guild_id, channel_id, self_deaf, self_mute); + let channel_id = channel_id.map(|c| c.0).map(From::from); + let cmd = TwilightVoiceState::new(guild_id.0, channel_id, self_deaf, self_mute); handle.command(*shard_id, &cmd).await?; Ok(()) }, #[cfg(feature = "twilight")] Shard::TwilightShard(handle) => { - let channel_id = nz_channel_id.map(From::from); - let cmd = TwilightVoiceState::new(nz_guild_id, channel_id, self_deaf, self_mute); + let channel_id = channel_id.map(|c| c.0).map(From::from); + let cmd = TwilightVoiceState::new(guild_id.0, channel_id, self_deaf, self_mute); handle.command(&cmd).await?; Ok(()) }, diff --git a/src/tracks/action.rs b/src/tracks/action.rs new file mode 100644 index 0000000..4d0d9a1 --- /dev/null +++ b/src/tracks/action.rs @@ -0,0 +1,43 @@ +use flume::Sender; +use std::time::Duration; + +use super::{PlayError, SeekRequest}; + +/// Actions for the mixer to take after inspecting track state via +/// [`TrackHandle::action`]. +/// +/// [`TrackHandle::action`]: super::TrackHandle::action +#[derive(Clone, Default)] +pub struct Action { + pub(crate) make_playable: Option>>, + pub(crate) seek_point: Option, +} + +impl Action { + /// Requests a seek to the given time for this track. + #[must_use] + pub fn seek(mut self, time: Duration) -> Self { + let (callback, _) = flume::bounded(1); + self.seek_point = Some(SeekRequest { time, callback }); + + self + } + + /// Readies the track to be playable, if this is not already the case. + #[must_use] + pub fn make_playable(mut self) -> Self { + let (tx, _) = flume::bounded(1); + self.make_playable = Some(tx); + + self + } + + pub(crate) fn combine(&mut self, other: Self) { + if other.make_playable.is_some() { + self.make_playable = other.make_playable; + } + if other.seek_point.is_some() { + self.seek_point = other.seek_point; + } + } +} diff --git a/src/tracks/command.rs b/src/tracks/command.rs index 1139512..0c3a17f 100644 --- a/src/tracks/command.rs +++ b/src/tracks/command.rs @@ -1,7 +1,10 @@ use super::*; use crate::events::EventData; use flume::Sender; -use std::time::Duration; +use std::{ + fmt::{Debug, Formatter, Result as FmtResult}, + time::Duration, +}; /// A request from external code using a [`TrackHandle`] to modify /// or act upon an [`Track`] object. @@ -21,37 +24,42 @@ pub enum TrackCommand { /// Seek to the given duration. /// /// On unsupported input types, this can be fatal. - Seek(Duration), + Seek(SeekRequest), /// Register an event on this track. AddEvent(EventData), /// Run some closure on this track, with direct access to the core object. - Do(Box), + Do(Box Option + Send + Sync + 'static>), /// Request a copy of this track's state. Request(Sender), /// Change the loop count/strategy of this track. Loop(LoopState), /// Prompts a track's input to become live and usable, if it is not already. - MakePlayable, + MakePlayable(Sender>), } -impl std::fmt::Debug for TrackCommand { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { - use TrackCommand::*; +impl Debug for TrackCommand { + fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult { write!( f, "TrackCommand::{}", match self { - Play => "Play".to_string(), - Pause => "Pause".to_string(), - Stop => "Stop".to_string(), - Volume(vol) => format!("Volume({})", vol), - Seek(d) => format!("Seek({:?})", d), - AddEvent(evt) => format!("AddEvent({:?})", evt), - Do(_f) => "Do([function])".to_string(), - Request(tx) => format!("Request({:?})", tx), - Loop(loops) => format!("Loop({:?})", loops), - MakePlayable => "MakePlayable".to_string(), + Self::Play => "Play".to_string(), + Self::Pause => "Pause".to_string(), + Self::Stop => "Stop".to_string(), + Self::Volume(vol) => format!("Volume({})", vol), + Self::Seek(s) => format!("Seek({:?})", s.time), + Self::AddEvent(evt) => format!("AddEvent({:?})", evt), + Self::Do(_f) => "Do([function])".to_string(), + Self::Request(tx) => format!("Request({:?})", tx), + Self::Loop(loops) => format!("Loop({:?})", loops), + Self::MakePlayable(_) => "MakePlayable".to_string(), } ) } } + +#[derive(Clone, Debug)] +pub struct SeekRequest { + pub time: Duration, + pub callback: Sender>, +} diff --git a/src/tracks/error.rs b/src/tracks/error.rs index 9cf3dc3..18e3eaa 100644 --- a/src/tracks/error.rs +++ b/src/tracks/error.rs @@ -1,40 +1,110 @@ -use std::{error::Error, fmt}; +use crate::input::AudioStreamError; +use flume::RecvError; +use std::{ + error::Error, + fmt::{Display, Formatter, Result as FmtResult}, + sync::Arc, +}; +use symphonia_core::errors::Error as SymphoniaError; /// Errors associated with control and manipulation of tracks. /// /// Unless otherwise stated, these don't invalidate an existing track, /// but do advise on valid operations and commands. -#[derive(Copy, Clone, Debug, Eq, PartialEq)] +#[derive(Clone, Debug)] #[non_exhaustive] -pub enum TrackError { +pub enum ControlError { /// The operation failed because the track has ended, has been removed /// due to call closure, or some error within the driver. Finished, /// The supplied event listener can never be fired by a track, and should /// be attached to the driver instead. InvalidTrackEvent, - /// The track's underlying [`Input`] doesn't support seeking operations. + /// A command to seek or ready the target track failed when parsing or creating the stream. /// - /// [`Input`]: crate::input::Input - SeekUnsupported, + /// This is a fatal error, and the track will be removed. + Play(PlayError), + /// Another `seek`/`make_playable` request was made, and so this callback handler was dropped. + Dropped, } -impl fmt::Display for TrackError { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { +impl Display for ControlError { + fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult { write!(f, "failed to operate on track (handle): ")?; match self { - TrackError::Finished => write!(f, "track ended"), - TrackError::InvalidTrackEvent => { + ControlError::Finished => write!(f, "track ended"), + ControlError::InvalidTrackEvent => { write!(f, "given event listener can't be fired on a track") }, - TrackError::SeekUnsupported => write!(f, "track did not support seeking"), + ControlError::Play(p) => { + write!(f, "i/o request on track failed: {}", p) + }, + ControlError::Dropped => write!(f, "request was replaced by another of same type"), } } } -impl Error for TrackError {} +impl Error for ControlError {} + +impl From for ControlError { + fn from(_: RecvError) -> Self { + ControlError::Dropped + } +} /// Alias for most calls to a [`TrackHandle`]. /// /// [`TrackHandle`]: super::TrackHandle -pub type TrackResult = Result; +pub type TrackResult = Result; + +/// Errors reported by the mixer while attempting to play (or ready) a [`Track`]. +/// +/// [`Track`]: super::Track +#[derive(Clone, Debug)] +#[non_exhaustive] +pub enum PlayError { + /// Failed to create a live bytestream from the lazy [`Compose`]. + /// + /// [`Compose`]: crate::input::Compose + Create(Arc), + /// Failed to read headers, codecs, or a valid stream from an [`Input`]. + /// + /// [`Input`]: crate::input::Input + Parse(Arc), + /// Failed to decode a frame received from an [`Input`]. + /// + /// [`Input`]: crate::input::Input + Decode(Arc), + /// Failed to seek to the requested location. + Seek(Arc), +} + +impl Display for PlayError { + fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult { + f.write_str("runtime error while playing track: ")?; + match self { + Self::Create(c) => { + f.write_str("input creation [")?; + f.write_fmt(format_args!("{}", &c))?; + f.write_str("]") + }, + Self::Parse(p) => { + f.write_str("parsing formats/codecs [")?; + f.write_fmt(format_args!("{}", &p))?; + f.write_str("]") + }, + Self::Decode(d) => { + f.write_str("decoding packets [")?; + f.write_fmt(format_args!("{}", &d))?; + f.write_str("]") + }, + Self::Seek(s) => { + f.write_str("seeking along input [")?; + f.write_fmt(format_args!("{}", &s))?; + f.write_str("]") + }, + } + } +} + +impl Error for PlayError {} diff --git a/src/tracks/handle.rs b/src/tracks/handle.rs index 173c7c8..ee893c7 100644 --- a/src/tracks/handle.rs +++ b/src/tracks/handle.rs @@ -1,9 +1,6 @@ use super::*; -use crate::{ - events::{Event, EventData, EventHandler}, - input::Metadata, -}; -use flume::Sender; +use crate::events::{Event, EventData, EventHandler}; +use flume::{Receiver, Sender}; use std::{fmt, sync::Arc, time::Duration}; use tokio::sync::RwLock; use typemap_rev::TypeMap; @@ -17,8 +14,7 @@ use uuid::Uuid; /// /// Many method calls here are fallible; in most cases, this will be because /// the underlying [`Track`] object has been discarded. Those which aren't refer -/// to immutable properties of the underlying stream, or shared data not used -/// by the driver. +/// to shared data not used by the driver. /// /// [`Track`]: Track pub struct TrackHandle { @@ -27,9 +23,7 @@ pub struct TrackHandle { struct InnerHandle { command_channel: Sender, - seekable: bool, uuid: Uuid, - metadata: Box, typemap: RwLock, } @@ -37,30 +31,21 @@ impl fmt::Debug for InnerHandle { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("InnerHandle") .field("command_channel", &self.command_channel) - .field("seekable", &self.seekable) .field("uuid", &self.uuid) - .field("metadata", &self.metadata) .field("typemap", &"") .finish() } } impl TrackHandle { - /// Creates a new handle, using the given command sink and hint as to whether - /// the underlying [`Input`] supports seek operations. + /// Creates a new handle, using the given command sink. /// /// [`Input`]: crate::input::Input - pub fn new( - command_channel: Sender, - seekable: bool, - uuid: Uuid, - metadata: Box, - ) -> Self { + #[must_use] + pub(crate) fn new(command_channel: Sender, uuid: Uuid) -> Self { let inner = Arc::new(InnerHandle { command_channel, - seekable, uuid, - metadata, typemap: RwLock::new(TypeMap::new()), }); @@ -92,53 +77,64 @@ impl TrackHandle { self.send(TrackCommand::Volume(volume)) } + #[must_use] /// Ready a track for playing if it is lazily initialised. /// - /// Currently, only [`Restartable`] sources support lazy setup. - /// This call is a no-op for all others. - /// - /// [`Restartable`]: crate::input::restartable::Restartable - pub fn make_playable(&self) -> TrackResult<()> { - self.send(TrackCommand::MakePlayable) + /// If a track is already playable, the callback will instantly succeed. + pub fn make_playable(&self) -> TrackCallback<()> { + let (tx, rx) = flume::bounded(1); + let fail = self.send(TrackCommand::MakePlayable(tx)).is_err(); + + TrackCallback { fail, rx } } - /// Denotes whether the underlying [`Input`] stream is compatible with arbitrary seeking. + /// Ready a track for playing if it is lazily initialised. /// - /// If this returns `false`, all calls to [`seek_time`] will fail, and the track is - /// incapable of looping. + /// This folds [`Self::make_playable`] into a single `async` result, but must + /// be awaited for the command to be sent. + pub async fn make_playable_async(&self) -> TrackResult<()> { + self.make_playable().result_async().await + } + + #[must_use] + /// Seeks along the track to the specified position. + /// + /// If the underlying [`Input`] does not support seeking, + /// forward seeks will succeed. Backward seeks will recreate the + /// track using the lazy [`Compose`] if present. The returned callback + /// will indicate whether the seek succeeded. /// - /// [`seek_time`]: TrackHandle::seek_time /// [`Input`]: crate::input::Input - pub fn is_seekable(&self) -> bool { - self.inner.seekable + /// [`Compose`]: crate::input::Compose + pub fn seek(&self, position: Duration) -> TrackCallback { + let (tx, rx) = flume::bounded(1); + let fail = self + .send(TrackCommand::Seek(SeekRequest { + time: position, + callback: tx, + })) + .is_err(); + + TrackCallback { fail, rx } } /// Seeks along the track to the specified position. /// - /// If the underlying [`Input`] does not support seeking, - /// then all calls will fail with [`TrackError::SeekUnsupported`]. - /// - /// [`Input`]: crate::input::Input - /// [`TrackError::SeekUnsupported`]: TrackError::SeekUnsupported - pub fn seek_time(&self, position: Duration) -> TrackResult<()> { - if self.is_seekable() { - self.send(TrackCommand::Seek(position)) - } else { - Err(TrackError::SeekUnsupported) - } + /// This folds [`Self::seek`] into a single `async` result, but must + /// be awaited for the command to be sent. + pub async fn seek_async(&self, position: Duration) -> TrackResult { + self.seek(position).result_async().await } /// Attach an event handler to an audio track. These will receive [`EventContext::Track`]. /// - /// Events which can only be fired by the global context return [`TrackError::InvalidTrackEvent`] + /// Events which can only be fired by the global context return [`ControlError::InvalidTrackEvent`] /// - /// [`Track`]: Track /// [`EventContext::Track`]: crate::events::EventContext::Track - /// [`TrackError::InvalidTrackEvent`]: TrackError::InvalidTrackEvent pub fn add_event(&self, event: Event, action: F) -> TrackResult<()> { let cmd = TrackCommand::AddEvent(EventData::new(event, action)); if event.is_global_only() { - Err(TrackError::InvalidTrackEvent) + Err(ControlError::InvalidTrackEvent) } else { self.send(cmd) } @@ -146,14 +142,17 @@ impl TrackHandle { /// Perform an arbitrary synchronous action on a raw [`Track`] object. /// + /// This will give access to a [`View`] of the current track state and [`Metadata`], + /// which can be used to take an [`Action`]. + /// /// Users **must** ensure that no costly work or blocking occurs /// within the supplied function or closure. *Taking excess time could prevent /// timely sending of packets, causing audio glitches and delays*. /// - /// [`Track`]: Track + /// [`Metadata`]: crate::input::Metadata pub fn action(&self, action: F) -> TrackResult<()> where - F: FnOnce(&mut Track) + Send + Sync + 'static, + F: FnOnce(View) -> Option + Send + Sync + 'static, { self.send(TrackCommand::Do(Box::new(action))) } @@ -163,77 +162,52 @@ impl TrackHandle { let (tx, rx) = flume::bounded(1); self.send(TrackCommand::Request(tx))?; - rx.recv_async().await.map_err(|_| TrackError::Finished) + rx.recv_async().await.map_err(|_| ControlError::Finished) } /// Set an audio track to loop indefinitely. /// - /// If the underlying [`Input`] does not support seeking, - /// then all calls will fail with [`TrackError::SeekUnsupported`]. + /// This requires either a [`Compose`] to be present or for the + /// input stream to be seekable. /// /// [`Input`]: crate::input::Input - /// [`TrackError::SeekUnsupported`]: TrackError::SeekUnsupported + /// [`Compose`]: crate::input::Compose pub fn enable_loop(&self) -> TrackResult<()> { - if self.is_seekable() { - self.send(TrackCommand::Loop(LoopState::Infinite)) - } else { - Err(TrackError::SeekUnsupported) - } + self.send(TrackCommand::Loop(LoopState::Infinite)) } /// Set an audio track to no longer loop. /// - /// If the underlying [`Input`] does not support seeking, - /// then all calls will fail with [`TrackError::SeekUnsupported`]. + /// This follows the same rules as [`enable_loop`]. /// - /// [`Input`]: crate::input::Input - /// [`TrackError::SeekUnsupported`]: TrackError::SeekUnsupported + /// [`enable_loop`]: Self::enable_loop pub fn disable_loop(&self) -> TrackResult<()> { - if self.is_seekable() { - self.send(TrackCommand::Loop(LoopState::Finite(0))) - } else { - Err(TrackError::SeekUnsupported) - } + self.send(TrackCommand::Loop(LoopState::Finite(0))) } /// Set an audio track to loop a set number of times. /// - /// If the underlying [`Input`] does not support seeking, - /// then all calls will fail with [`TrackError::SeekUnsupported`]. + /// This follows the same rules as [`enable_loop`]. /// - /// [`Input`]: crate::input::Input - /// [`TrackError::SeekUnsupported`]: TrackError::SeekUnsupported + /// [`enable_loop`]: Self::enable_loop pub fn loop_for(&self, count: usize) -> TrackResult<()> { - if self.is_seekable() { - self.send(TrackCommand::Loop(LoopState::Finite(count))) - } else { - Err(TrackError::SeekUnsupported) - } + self.send(TrackCommand::Loop(LoopState::Finite(count))) } /// Returns this handle's (and track's) unique identifier. + #[must_use] pub fn uuid(&self) -> Uuid { self.inner.uuid } - /// Returns the metadata stored in the handle. + /// Allows access to this track's attached [`TypeMap`]. /// - /// Metadata is cloned from the inner [`Input`] at - /// the time a track/handle is created, and is effectively - /// read-only from then on. - /// - /// [`Input`]: crate::input::Input - pub fn metadata(&self) -> &Metadata { - &self.inner.metadata - } - - /// Allows access to this track's attached TypeMap. - /// - /// TypeMaps allow additional, user-defined data shared by all handles + /// [`TypeMap`]s allow additional, user-defined data shared by all handles /// to be attached to any track. /// /// Driver code will never attempt to lock access to this map, /// preventing deadlock/stalling. + #[must_use] pub fn typemap(&self) -> &RwLock { &self.inner.typemap } @@ -242,12 +216,95 @@ impl TrackHandle { /// Send a raw command to the [`Track`] object. /// /// [`Track`]: Track - pub fn send(&self, cmd: TrackCommand) -> TrackResult<()> { + pub(crate) fn send(&self, cmd: TrackCommand) -> TrackResult<()> { // As the send channels are unbounded, we can be reasonably certain // that send failure == cancellation. self.inner .command_channel .send(cmd) - .map_err(|_e| TrackError::Finished) + .map_err(|_e| ControlError::Finished) + } +} + +/// Asynchronous reply for an operation applied to a [`TrackHandle`]. +/// +/// This object does not need to be `.await`ed for the driver to perform an action. +/// Async threads can then call, e.g., [`TrackHandle::make_playable`], and safely drop +/// this callback if the result isn't needed. +pub struct TrackCallback { + fail: bool, + rx: Receiver>, +} + +impl TrackCallback { + /// Consumes this handle to await a reply from the driver, blocking the current thread. + pub fn result(self) -> TrackResult { + if self.fail { + Err(ControlError::Finished) + } else { + self.rx.recv()?.map_err(ControlError::Play) + } + } + + /// Consumes this handle to await a reply from the driver asynchronously. + pub async fn result_async(self) -> TrackResult { + if self.fail { + Err(ControlError::Finished) + } else { + self.rx.recv_async().await?.map_err(ControlError::Play) + } + } + + #[must_use] + /// Returns `true` if the operation instantly failed due to the target track being + /// removed. + pub fn is_hung_up(&self) -> bool { + self.fail + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + constants::test_data::FILE_WAV_TARGET, + driver::Driver, + input::File, + tracks::Track, + Config, + }; + + #[tokio::test] + #[ntest::timeout(10_000)] + async fn make_playable_callback_fires() { + let (t_handle, config) = Config::test_cfg(true); + let mut driver = Driver::new(config.clone()); + + let file = File::new(FILE_WAV_TARGET); + let handle = driver.play(Track::from(file).pause()); + + let callback = handle.make_playable(); + t_handle.spawn_ticker().await; + assert!(callback.result_async().await.is_ok()); + } + + #[tokio::test] + #[ntest::timeout(10_000)] + async fn seek_callback_fires() { + let (t_handle, config) = Config::test_cfg(true); + let mut driver = Driver::new(config.clone()); + + let file = File::new(FILE_WAV_TARGET); + let handle = driver.play(Track::from(file).pause()); + + let target = Duration::from_millis(500); + let callback = handle.seek(target); + t_handle.spawn_ticker().await; + + let answer = callback.result_async().await; + assert!(answer.is_ok()); + let answer = answer.unwrap(); + let delta = Duration::from_millis(100); + assert!(answer > target - delta && answer < target + delta); } } diff --git a/src/tracks/looping.rs b/src/tracks/looping.rs index 42c03ba..80be32c 100644 --- a/src/tracks/looping.rs +++ b/src/tracks/looping.rs @@ -20,3 +20,109 @@ impl Default for LoopState { Self::Finite(0) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + constants::test_data::FILE_WAV_TARGET, + driver::Driver, + input::File, + tracks::{PlayMode, Track, TrackState}, + Config, + Event, + EventContext, + EventHandler, + TrackEvent, + }; + use flume::Sender; + + struct Looper { + tx: Sender, + } + + #[async_trait::async_trait] + impl EventHandler for Looper { + async fn act(&self, ctx: &crate::EventContext<'_>) -> Option { + if let EventContext::Track(&[(state, _)]) = ctx { + drop(self.tx.send(state.clone())); + } + + None + } + } + + #[tokio::test] + #[ntest::timeout(10_000)] + async fn finite_track_loops_work() { + let (t_handle, config) = Config::test_cfg(true); + let mut driver = Driver::new(config.clone()); + + let file = File::new(FILE_WAV_TARGET); + let handle = driver.play(Track::from(file).loops(LoopState::Finite(2))); + + let (l_tx, l_rx) = flume::unbounded(); + let (e_tx, e_rx) = flume::unbounded(); + let _ = handle.add_event(Event::Track(TrackEvent::Loop), Looper { tx: l_tx }); + let _ = handle.add_event(Event::Track(TrackEvent::End), Looper { tx: e_tx }); + + t_handle.spawn_ticker().await; + + // CONDITIONS: + // 1) 2 loop events, each changes the loop count. + // 2) Track ends. + // 3) Playtime >> Position + assert_eq!( + l_rx.recv_async().await.map(|v| v.loops), + Ok(LoopState::Finite(1)) + ); + assert_eq!( + l_rx.recv_async().await.map(|v| v.loops), + Ok(LoopState::Finite(0)) + ); + let ended = e_rx.recv_async().await; + + assert!(ended.is_ok()); + + let ended = ended.unwrap(); + assert!(ended.play_time > 2 * ended.position); + } + + #[tokio::test] + #[ntest::timeout(10_000)] + async fn infinite_track_loops_work() { + let (t_handle, config) = Config::test_cfg(true); + let mut driver = Driver::new(config.clone()); + + let file = File::new(FILE_WAV_TARGET); + let handle = driver.play(Track::from(file).loops(LoopState::Infinite)); + + let (l_tx, l_rx) = flume::unbounded(); + let _ = handle.add_event(Event::Track(TrackEvent::Loop), Looper { tx: l_tx }); + + t_handle.spawn_ticker().await; + + // CONDITIONS: + // 1) 3 loop events, each does not change the loop count. + // 2) Track still playing at final + // 3) Playtime >> Position + assert_eq!( + l_rx.recv_async().await.map(|v| v.loops), + Ok(LoopState::Infinite) + ); + assert_eq!( + l_rx.recv_async().await.map(|v| v.loops), + Ok(LoopState::Infinite) + ); + + let final_state = l_rx.recv_async().await; + assert_eq!( + final_state.as_ref().map(|v| v.loops), + Ok(LoopState::Infinite) + ); + let final_state = final_state.unwrap(); + + assert_eq!(final_state.playing, PlayMode::Play); + assert!(final_state.play_time > 2 * final_state.position); + } +} diff --git a/src/tracks/mod.rs b/src/tracks/mod.rs index 90ac096..3d175c2 100644 --- a/src/tracks/mod.rs +++ b/src/tracks/mod.rs @@ -14,383 +14,189 @@ //! [`TrackHandle`]: struct.TrackHandle.html //! [`create_player`]: fn.create_player.html +mod action; mod command; mod error; mod handle; mod looping; mod mode; mod queue; +mod ready; mod state; +mod view; -pub use self::{command::*, error::*, handle::*, looping::*, mode::*, queue::*, state::*}; +pub use self::{ + action::*, + error::*, + handle::*, + looping::*, + mode::*, + queue::*, + ready::*, + state::*, + view::*, +}; +pub(crate) use command::*; use crate::{constants::*, driver::tasks::message::*, events::EventStore, input::Input}; -use flume::{Receiver, TryRecvError}; use std::time::Duration; use uuid::Uuid; -/// Control object for audio playback. +/// Initial state for audio playback. /// -/// Accessed by both commands and the playback code -- as such, access from user code is -/// almost always guarded via a [`TrackHandle`]. You should expect to receive -/// access to a raw object of this type via [`create_player`], for use in -/// [`Driver::play`] or [`Driver::play_only`]. +/// [`Track`]s allow you to configure play modes, volume, event handlers, and other track state +/// before you pass an input to the [`Driver`]. +/// +/// Live track data is accesseed via a [`TrackHandle`], returned by [`Driver::play`] and +/// related methods. /// /// # Example /// /// ```rust,no_run -/// use songbird::{driver::Driver, ffmpeg, tracks::create_player}; +/// use songbird::{driver::Driver, input::File, tracks::Track}; /// /// # async { /// // A Call is also valid here! /// let mut handler: Driver = Default::default(); -/// let source = ffmpeg("../audio/my-favourite-song.mp3") -/// .await -/// .expect("This might fail: handle this error!"); -/// let (mut audio, audio_handle) = create_player(source); +/// let source = File::new("../audio/my-favourite-song.mp3"); /// -/// audio.set_volume(0.5); -/// -/// handler.play_only(audio); +/// handler.play_only(Track::new(source.into()).volume(0.5)); /// /// // Future access occurs via audio_handle. /// # }; /// ``` /// -/// [`Driver::play_only`]: crate::driver::Driver::play_only +/// [`Driver`]: crate::driver::Driver /// [`Driver::play`]: crate::driver::Driver::play -/// [`TrackHandle`]: TrackHandle -/// [`create_player`]: create_player -#[derive(Debug)] pub struct Track { /// Whether or not this sound is currently playing. /// - /// Can be controlled with [`play`] or [`pause`] if chaining is desired. - /// - /// [`play`]: Track::play - /// [`pause`]: Track::pause - pub(crate) playing: PlayMode, + /// Defaults to [`PlayMode::Play`]. + pub playing: PlayMode, - /// The desired volume for playback. + /// The volume for playback. /// - /// Sensible values fall between `0.0` and `1.0`. + /// Sensible values fall between `0.0` and `1.0`. Values outside this range can + /// cause clipping or other audio artefacts. /// - /// Can be controlled with [`volume`] if chaining is desired. - /// - /// [`volume`]: Track::volume - pub(crate) volume: f32, + /// Defaults to `1.0`. + pub volume: f32, - /// Underlying data access object. - /// - /// *Calling code is not expected to use this.* - pub(crate) source: Input, - - /// The current playback position in the track. - pub(crate) position: Duration, - - /// The total length of time this track has been active. - pub(crate) play_time: Duration, + /// The live or lazily-initialised audio stream to be played. + pub input: Input, /// List of events attached to this audio track. /// /// This may be used to add additional events to a track /// before it is sent to the audio context for playing. - pub events: Option, - - /// Channel from which commands are received. /// - /// Track commands are sent in this manner to ensure that access - /// occurs in a thread-safe manner, without allowing any external - /// code to lock access to audio objects and block packet generation. - pub(crate) commands: Receiver, - - /// Handle for safe control of this audio track from other threads. - /// - /// Typically, this is used by internal code to supply context information - /// to event handlers, though more may be cloned from this handle. - pub handle: TrackHandle, + /// Defaults to an empty set. + pub events: EventStore, /// Count of remaining loops. + /// + /// Defaults to play a track once (i.e., [`LoopState::Finite(0)`]). + /// + /// [`LoopState::Finite(0)`]: LoopState::Finite pub loops: LoopState, /// Unique identifier for this track. - pub(crate) uuid: Uuid, + /// + /// Defaults to a random 128-bit number. + pub uuid: Uuid, } impl Track { - /// Create a new track directly from an input, command source, - /// and handle. - /// - /// In general, you should probably use [`create_player`]. - /// - /// [`create_player`]: fn.create_player.html - pub fn new_raw(source: Input, commands: Receiver, handle: TrackHandle) -> Self { - let uuid = handle.uuid(); + /// Create a new track directly from an [`Input`] and a random [`Uuid`]. + #[must_use] + pub fn new(input: Input) -> Self { + let uuid = Uuid::new_v4(); + Self::new_with_uuid(input, uuid) + } + + /// Create a new track directly from an [`Input`] with a custom [`Uuid`]. + #[must_use] + pub fn new_with_uuid(input: Input, uuid: Uuid) -> Self { Self { - playing: Default::default(), + playing: PlayMode::default(), volume: 1.0, - source, - position: Default::default(), - play_time: Default::default(), - events: Some(EventStore::new_local()), - commands, - handle, + input, + events: EventStore::new_local(), loops: LoopState::Finite(0), uuid, } } + #[must_use] /// Sets a track to playing if it is paused. - pub fn play(&mut self) -> &mut Self { - self.set_playing(PlayMode::Play) - } - - /// Pauses a track if it is playing. - pub fn pause(&mut self) -> &mut Self { - self.set_playing(PlayMode::Pause) - } - - /// Manually stops a track. - /// - /// This will cause the audio track to be removed, with any relevant events triggered. - /// Stopped/ended tracks cannot be restarted. - pub fn stop(&mut self) -> &mut Self { - self.set_playing(PlayMode::Stop) - } - - pub(crate) fn end(&mut self) -> &mut Self { - self.set_playing(PlayMode::End) - } - - #[inline] - fn set_playing(&mut self, new_state: PlayMode) -> &mut Self { - self.playing = self.playing.change_to(new_state); - + pub fn play(mut self) -> Self { + self.playing = PlayMode::Play; self } - /// Returns the current play status of this track. - pub fn playing(&self) -> PlayMode { - self.playing + #[must_use] + /// Pre-emptively pauses a track, preventing it from being automatically played. + pub fn pause(mut self) -> Self { + self.playing = PlayMode::Pause; + self } + #[must_use] + /// Manually stops a track. + /// + /// This will cause the audio track to be removed by the driver almost immediately, + /// with any relevant events triggered. + pub fn stop(mut self) -> Self { + self.playing = PlayMode::Stop; + self + } + + #[must_use] /// Sets [`volume`] in a manner that allows method chaining. /// /// [`volume`]: Track::volume - pub fn set_volume(&mut self, volume: f32) -> &mut Self { + pub fn volume(mut self, volume: f32) -> Self { self.volume = volume; self } - /// Returns the current playback position. - pub fn volume(&self) -> f32 { - self.volume - } - - /// Returns the current playback position. - pub fn position(&self) -> Duration { - self.position - } - - /// Returns the total length of time this track has been active. - pub fn play_time(&self) -> Duration { - self.play_time - } - + #[must_use] /// Set an audio track to loop a set number of times. - /// - /// If the underlying [`Input`] does not support seeking, - /// then all calls will fail with [`TrackError::SeekUnsupported`]. - /// - /// [`Input`]: crate::input::Input - /// [`TrackError::SeekUnsupported`]: TrackError::SeekUnsupported - pub fn set_loops(&mut self, loops: LoopState) -> TrackResult<()> { - if self.source.is_seekable() { - self.loops = loops; - Ok(()) - } else { - Err(TrackError::SeekUnsupported) - } - } - - pub(crate) fn do_loop(&mut self) -> bool { - match self.loops { - LoopState::Infinite => true, - LoopState::Finite(0) => false, - LoopState::Finite(ref mut n) => { - *n -= 1; - true - }, - } - } - - /// Steps playback location forward by one frame. - pub(crate) fn step_frame(&mut self) { - self.position += TIMESTEP_LENGTH; - self.play_time += TIMESTEP_LENGTH; - } - - /// Receives and acts upon any commands forwarded by TrackHandles. - /// - /// *Used internally*, this should not be exposed to users. - pub(crate) fn process_commands(&mut self, index: usize, ic: &Interconnect) { - // Note: disconnection and an empty channel are both valid, - // and should allow the audio object to keep running as intended. - - // Note that interconnect failures are not currently errors. - // In correct operation, the event thread should never panic, - // but it receiving status updates is secondary do actually - // doing the work. - loop { - match self.commands.try_recv() { - Ok(cmd) => { - use TrackCommand::*; - match cmd { - Play => { - self.play(); - let _ = ic.events.send(EventMessage::ChangeState( - index, - TrackStateChange::Mode(self.playing), - )); - }, - Pause => { - self.pause(); - let _ = ic.events.send(EventMessage::ChangeState( - index, - TrackStateChange::Mode(self.playing), - )); - }, - Stop => { - self.stop(); - let _ = ic.events.send(EventMessage::ChangeState( - index, - TrackStateChange::Mode(self.playing), - )); - }, - Volume(vol) => { - self.set_volume(vol); - let _ = ic.events.send(EventMessage::ChangeState( - index, - TrackStateChange::Volume(self.volume), - )); - }, - Seek(time) => - if let Ok(new_time) = self.seek_time(time) { - let _ = ic.events.send(EventMessage::ChangeState( - index, - TrackStateChange::Position(new_time), - )); - }, - AddEvent(evt) => { - let _ = ic.events.send(EventMessage::AddTrackEvent(index, evt)); - }, - Do(action) => { - action(self); - let _ = ic.events.send(EventMessage::ChangeState( - index, - TrackStateChange::Total(self.state()), - )); - }, - Request(tx) => { - let _ = tx.send(self.state()); - }, - Loop(loops) => - if self.set_loops(loops).is_ok() { - let _ = ic.events.send(EventMessage::ChangeState( - index, - TrackStateChange::Loops(self.loops, true), - )); - }, - MakePlayable => self.make_playable(), - } - }, - Err(TryRecvError::Disconnected) => { - // this branch will never be visited. - break; - }, - Err(TryRecvError::Empty) => { - break; - }, - } - } - } - - /// Ready a track for playing if it is lazily initialised. - /// - /// Currently, only [`Restartable`] sources support lazy setup. - /// This call is a no-op for all others. - /// - /// [`Restartable`]: crate::input::restartable::Restartable - pub fn make_playable(&mut self) { - self.source.reader.make_playable(); - } - - /// Creates a read-only copy of the audio track's state. - /// - /// The primary use-case of this is sending information across - /// threads in response to a [`TrackHandle`]. - /// - /// [`TrackHandle`]: TrackHandle - pub fn state(&self) -> TrackState { - TrackState { - playing: self.playing, - volume: self.volume, - position: self.position, - play_time: self.play_time, - loops: self.loops, - } - } - - /// Seek to a specific point in the track. - /// - /// If the underlying [`Input`] does not support seeking, - /// then all calls will fail with [`TrackError::SeekUnsupported`]. - /// - /// [`Input`]: crate::input::Input - /// [`TrackError::SeekUnsupported`]: TrackError::SeekUnsupported - pub fn seek_time(&mut self, pos: Duration) -> TrackResult { - if let Some(t) = self.source.seek_time(pos) { - self.position = t; - Ok(t) - } else { - Err(TrackError::SeekUnsupported) - } + pub fn loops(mut self, loops: LoopState) -> Self { + self.loops = loops; + + self } + #[must_use] /// Returns this track's unique identifier. - pub fn uuid(&self) -> Uuid { - self.uuid + pub fn uuid(mut self, uuid: Uuid) -> Self { + self.uuid = uuid; + + self + } + + pub(crate) fn into_context(self) -> (TrackHandle, TrackContext) { + let (tx, receiver) = flume::unbounded(); + let handle = TrackHandle::new(tx, self.uuid); + + let context = TrackContext { + handle: handle.clone(), + track: self, + receiver, + }; + + (handle, context) } } -/// Creates a [`Track`] object to pass into the audio context, and a [`TrackHandle`] -/// for safe, lock-free access in external code. -/// -/// Typically, this would be used if you wished to directly work on or configure -/// the [`Track`] object before it is passed over to the driver. -/// -/// [`Track`]: Track -/// [`TrackHandle`]: TrackHandle -#[inline] -pub fn create_player(source: Input) -> (Track, TrackHandle) { - create_player_with_uuid(source, Uuid::new_v4()) -} - -/// Creates a [`Track`] and [`TrackHandle`] as in [`create_player`], allowing -/// a custom UUID to be set. -/// -/// [`create_player`]: create_player -/// [`Track`]: Track -/// [`TrackHandle`]: TrackHandle -pub fn create_player_with_uuid(source: Input, uuid: Uuid) -> (Track, TrackHandle) { - let (tx, rx) = flume::unbounded(); - let can_seek = source.is_seekable(); - let metadata = source.metadata.clone(); - let handle = TrackHandle::new(tx, can_seek, uuid, metadata); - - let player = Track::new_raw(source, rx, handle.clone()); - - (player, handle) +/// Any [`Input`] (or struct which can be used as one) can also be made into a [`Track`]. +impl> From for Track { + // NOTE: this is `Into` to support user-given structs which can + // only `impl Into`. + fn from(val: T) -> Self { + Track::new(val.into()) + } } diff --git a/src/tracks/mode.rs b/src/tracks/mode.rs index b54f6ae..e7a2162 100644 --- a/src/tracks/mode.rs +++ b/src/tracks/mode.rs @@ -1,7 +1,8 @@ +use super::PlayError; use crate::events::TrackEvent; /// Playback status of a track. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[derive(Clone, Debug)] #[non_exhaustive] pub enum PlayMode { /// The track is currently playing. @@ -12,32 +13,56 @@ pub enum PlayMode { Stop, /// The track has naturally ended, and cannot be restarted. End, + /// The track has encountered a runtime or initialisation error, and cannot be restarted. + Errored(PlayError), } impl PlayMode { /// Returns whether the track has irreversibly stopped. - pub fn is_done(self) -> bool { - matches!(self, PlayMode::Stop | PlayMode::End) + #[must_use] + pub fn is_done(&self) -> bool { + matches!(self, PlayMode::Stop | PlayMode::End | PlayMode::Errored(_)) } - pub(crate) fn change_to(self, other: Self) -> PlayMode { - use PlayMode::*; + /// Returns whether the track has irreversibly stopped. + #[must_use] + pub(crate) fn is_playing(&self) -> bool { + matches!(self, PlayMode::Play) + } + #[must_use] + pub(crate) fn next_state(self, other: Self) -> Self { // Idea: a finished track cannot be restarted -- this action is final. // We may want to change this in future so that seekable tracks can uncancel // themselves, perhaps, but this requires a bit more machinery to readd... match self { - Play | Pause => other, + Self::Play | Self::Pause => other, state => state, } } - pub(crate) fn as_track_event(self) -> TrackEvent { - use PlayMode::*; + pub(crate) fn change_to(&mut self, other: Self) { + *self = self.clone().next_state(other); + } + + #[must_use] + pub(crate) fn as_track_event(&self) -> TrackEvent { match self { - Play => TrackEvent::Play, - Pause => TrackEvent::Pause, - Stop | End => TrackEvent::End, + Self::Play => TrackEvent::Play, + Self::Pause => TrackEvent::Pause, + Self::Stop | Self::End => TrackEvent::End, + Self::Errored(_) => TrackEvent::Error, + } + } + + // The above fn COULD just return a Vec, but the below means we only allocate a Vec + // in the rare error case. + // Also, see discussion on bitsets in src/events/track.rs + #[must_use] + pub(crate) fn also_fired_track_events(&self) -> Option> { + match self { + Self::Errored(_) => Some(vec![TrackEvent::End]), + _ => None, } } } @@ -47,3 +72,11 @@ impl Default for PlayMode { PlayMode::Play } } + +impl PartialEq for PlayMode { + fn eq(&self, other: &Self) -> bool { + self.as_track_event() == other.as_track_event() + } +} + +impl Eq for PlayMode {} diff --git a/src/tracks/queue.rs b/src/tracks/queue.rs index 0ad6bf5..286e2de 100644 --- a/src/tracks/queue.rs +++ b/src/tracks/queue.rs @@ -2,7 +2,7 @@ use crate::{ driver::Driver, events::{Event, EventContext, EventData, EventHandler, TrackEvent}, input::Input, - tracks::{self, Track, TrackHandle, TrackResult}, + tracks::{Track, TrackHandle, TrackResult}, }; use async_trait::async_trait; use parking_lot::Mutex; @@ -28,28 +28,27 @@ use tracing::{info, warn}; /// use songbird::{ /// driver::Driver, /// id::GuildId, -/// ffmpeg, -/// tracks::{create_player, TrackQueue}, +/// input::File, +/// tracks::TrackQueue, /// }; /// use std::collections::HashMap; +/// use std::num::NonZeroU64; /// /// # async { -/// let guild = GuildId(0); +/// let guild = GuildId(NonZeroU64::new(1).unwrap()); /// // A Call is also valid here! /// let mut driver: Driver = Default::default(); /// /// let mut queues: HashMap = Default::default(); /// -/// let source = ffmpeg("../audio/my-favourite-song.mp3") -/// .await -/// .expect("This might fail: handle this error!"); +/// let source = File::new("../audio/my-favourite-song.mp3"); /// /// // We need to ensure that this guild has a TrackQueue created for it. /// let queue = queues.entry(guild) /// .or_default(); /// /// // Queueing a track is this easy! -/// queue.add_source(source, &mut driver); +/// queue.add_source(source.into(), &mut driver); /// # }; /// ``` /// @@ -77,6 +76,7 @@ impl Deref for Queued { impl Queued { /// Clones the inner handle + #[must_use] pub fn handle(&self) -> TrackHandle { self.0.clone() } @@ -147,7 +147,9 @@ impl EventHandler for SongPreloader { let inner = self.remote_lock.lock(); if let Some(track) = inner.tracks.get(1) { - let _ = track.0.make_playable(); + // This is the sync-version so that we can fire and ignore + // the request ASAP. + drop(track.0.make_playable()); } None @@ -156,6 +158,7 @@ impl EventHandler for SongPreloader { impl TrackQueue { /// Create a new, empty, track queue. + #[must_use] pub fn new() -> Self { Self { inner: Arc::new(Mutex::new(TrackQueueCore { @@ -164,72 +167,99 @@ impl TrackQueue { } } - /// Adds an audio source to the queue, to be played in the channel managed by `handler`. - pub fn add_source(&self, source: Input, handler: &mut Driver) -> TrackHandle { - let (track, handle) = tracks::create_player(source); - self.add(track, handler); + /// Adds an audio source to the queue, to be played in the channel managed by `driver`. + /// + /// This method will preload the next track 5 seconds before the current track ends, if + /// the [`AuxMetadata`] can be successfully queried for a [`Duration`]. + /// + /// [`AuxMetadata`]: crate::input::AuxMetadata + pub async fn add_source(&self, input: Input, driver: &mut Driver) -> TrackHandle { + self.add(input.into(), driver).await + } + + /// Adds a [`Track`] object to the queue, to be played in the channel managed by `driver`. + /// + /// This allows additional configuration or event handlers to be added + /// before enqueueing the audio track. [`Track`]s will be paused pre-emptively. + /// + /// This method will preload the next track 5 seconds before the current track ends, if + /// the [`AuxMetadata`] can be successfully queried for a [`Duration`]. + /// + /// [`AuxMetadata`]: crate::input::AuxMetadata + pub async fn add(&self, mut track: Track, driver: &mut Driver) -> TrackHandle { + let preload_time = Self::get_preload_time(&mut track).await; + self.add_with_preload(track, driver, preload_time) + } + + pub(crate) async fn get_preload_time(track: &mut Track) -> Option { + let meta = match track.input { + Input::Lazy(ref mut rec) => rec.aux_metadata().await.ok(), + Input::Live(_, Some(ref mut rec)) => rec.aux_metadata().await.ok(), + Input::Live(_, None) => None, + }; + + meta.and_then(|meta| meta.duration) + .map(|d| d.saturating_sub(Duration::from_secs(5))) + } + + /// Add an existing [`Track`] to the queue, using a known time to preload the next track. + /// + /// `preload_time` can be specified to enable gapless playback: this is the + /// playback position *in this track* when the the driver will begin to load the next track. + /// The standard [`Self::add`] method use [`AuxMetadata`] to set this to 5 seconds before + /// a track ends. + /// + /// A `None` value will not ready the next track until this track ends, disabling preload. + /// + /// [`AuxMetadata`]: crate::input::AuxMetadata + #[inline] + pub fn add_with_preload( + &self, + mut track: Track, + driver: &mut Driver, + preload_time: Option, + ) -> TrackHandle { + // Attempts to start loading the next track before this one ends. + // Idea is to provide as close to gapless playback as possible, + // while minimising memory use. + info!("Track added to queue."); + + let remote_lock = self.inner.clone(); + track.events.add_event( + EventData::new(Event::Track(TrackEvent::End), QueueHandler { remote_lock }), + Duration::ZERO, + ); + + if let Some(time) = preload_time { + let remote_lock = self.inner.clone(); + track.events.add_event( + EventData::new(Event::Delayed(time), SongPreloader { remote_lock }), + Duration::ZERO, + ); + } + + let (should_play, handle) = { + let mut inner = self.inner.lock(); + + let handle = driver.play(track.pause()); + inner.tracks.push_back(Queued(handle.clone())); + + (inner.tracks.len() == 1, handle) + }; + + if should_play { + drop(handle.play()); + } handle } - /// Adds a [`Track`] object to the queue, to be played in the channel managed by `handler`. - /// - /// This is used with [`create_player`] if additional configuration or event handlers - /// are required before enqueueing the audio track. - /// - /// [`Track`]: Track - /// [`create_player`]: super::create_player - pub fn add(&self, mut track: Track, handler: &mut Driver) { - self.add_raw(&mut track); - handler.play(track); - } - - #[inline] - pub(crate) fn add_raw(&self, track: &mut Track) { - info!("Track added to queue."); - let remote_lock = self.inner.clone(); - let mut inner = self.inner.lock(); - - let track_handle = track.handle.clone(); - - if !inner.tracks.is_empty() { - track.pause(); - } - - track - .events - .as_mut() - .expect("Queue inspecting EventStore on new Track: did not exist.") - .add_event( - EventData::new(Event::Track(TrackEvent::End), QueueHandler { remote_lock }), - track.position, - ); - - // Attempts to start loading the next track before this one ends. - // Idea is to provide as close to gapless playback as possible, - // while minimising memory use. - if let Some(time) = track.source.metadata.duration { - let preload_time = time.checked_sub(Duration::from_secs(5)).unwrap_or_default(); - let remote_lock = self.inner.clone(); - - track - .events - .as_mut() - .expect("Queue inspecting EventStore on new Track: did not exist.") - .add_event( - EventData::new(Event::Delayed(preload_time), SongPreloader { remote_lock }), - track.position, - ); - } - - inner.tracks.push_back(Queued(track_handle)); - } - /// Returns a handle to the currently playing track. + #[must_use] pub fn current(&self) -> Option { let inner = self.inner.lock(); - inner.tracks.front().map(|h| h.handle()) + inner.tracks.front().map(Queued::handle) } /// Attempts to remove a track from the specified index. @@ -237,11 +267,13 @@ impl TrackQueue { /// The returned entry can be readded to *this* queue via [`modify_queue`]. /// /// [`modify_queue`]: TrackQueue::modify_queue + #[must_use] pub fn dequeue(&self, index: usize) -> Option { self.modify_queue(|vq| vq.remove(index)) } /// Returns the number of tracks currently in the queue. + #[must_use] pub fn len(&self) -> usize { let inner = self.inner.lock(); @@ -249,6 +281,7 @@ impl TrackQueue { } /// Returns whether there are no tracks currently in the queue. + #[must_use] pub fn is_empty(&self) -> bool { let inner = self.inner.lock(); @@ -296,7 +329,7 @@ impl TrackQueue { for track in inner.tracks.drain(..) { // Errors when removing tracks don't really make // a difference: an error just implies it's already gone. - let _ = track.stop(); + drop(track.stop()); } } @@ -314,10 +347,11 @@ impl TrackQueue { /// Use [`modify_queue`] for direct modification of the queue. /// /// [`modify_queue`]: TrackQueue::modify_queue + #[must_use] pub fn current_queue(&self) -> Vec { let inner = self.inner.lock(); - inner.tracks.iter().map(|q| q.handle()).collect() + inner.tracks.iter().map(Queued::handle).collect() } } @@ -331,3 +365,135 @@ impl TrackQueueCore { } } } + +#[cfg(all(test, feature = "builtin-queue"))] +mod tests { + use crate::{ + driver::Driver, + input::{File, HttpRequest}, + tracks::PlayMode, + Config, + }; + use reqwest::Client; + use std::time::Duration; + + #[tokio::test] + #[ntest::timeout(20_000)] + async fn next_track_plays_on_end() { + let (t_handle, config) = Config::test_cfg(true); + let mut driver = Driver::new(config.clone()); + + let file1 = File::new("resources/ting.wav"); + let file2 = file1.clone(); + + let h1 = driver.enqueue_input(file1.into()).await; + let h2 = driver.enqueue_input(file2.into()).await; + + // Get h1 in place, playing. Wait for IO to ready. + // Fast wait here since it's all local I/O, no network. + t_handle + .ready_track(&h1, Some(Duration::from_millis(1))) + .await; + t_handle + .ready_track(&h2, Some(Duration::from_millis(1))) + .await; + + // playout + t_handle.tick(1); + t_handle.wait(1); + + let h1a = h1.get_info(); + let h2a = h2.get_info(); + + // allow get_info to fire for h2. + t_handle.tick(2); + + // post-conditions: + // 1) track 1 is done & dropped (commands fail). + // 2) track 2 is playing. + assert!(h1a.await.is_err()); + assert_eq!(h2a.await.unwrap().playing, PlayMode::Play); + } + + #[tokio::test] + #[ntest::timeout(10_000)] + async fn next_track_plays_on_skip() { + let (t_handle, config) = Config::test_cfg(true); + let mut driver = Driver::new(config.clone()); + + let file1 = File::new("resources/ting.wav"); + let file2 = file1.clone(); + + let h1 = driver.enqueue_input(file1.into()).await; + let h2 = driver.enqueue_input(file2.into()).await; + + // Get h1 in place, playing. Wait for IO to ready. + // Fast wait here since it's all local I/O, no network. + t_handle + .ready_track(&h1, Some(Duration::from_millis(1))) + .await; + + assert!(driver.queue().skip().is_ok()); + + t_handle + .ready_track(&h2, Some(Duration::from_millis(1))) + .await; + + // playout + t_handle.skip(1).await; + + let h1a = h1.get_info(); + let h2a = h2.get_info(); + + // allow get_info to fire for h2. + t_handle.tick(2); + + // post-conditions: + // 1) track 1 is done & dropped (commands fail). + // 2) track 2 is playing. + assert!(h1a.await.is_err()); + assert_eq!(h2a.await.unwrap().playing, PlayMode::Play); + } + + #[tokio::test] + #[ntest::timeout(10_000)] + async fn next_track_plays_on_err() { + let (t_handle, config) = Config::test_cfg(true); + let mut driver = Driver::new(config.clone()); + + // File 1 is HTML with no valid audio -- this will fail to play. + let file1 = HttpRequest::new( + Client::new(), + "http://github.com/serenity-rs/songbird/".into(), + ); + let file2 = File::new("resources/ting.wav"); + + let h1 = driver.enqueue_input(file1.into()).await; + let h2 = driver.enqueue_input(file2.into()).await; + + // Get h1 in place, playing. Wait for IO to ready. + // Fast wait here since it's all local I/O, no network. + // t_handle + // .ready_track(&h1, Some(Duration::from_millis(1))) + // .await; + t_handle + .ready_track(&h2, Some(Duration::from_millis(1))) + .await; + + // playout + t_handle.tick(1); + t_handle.wait(1); + + let h1a = h1.get_info(); + let h2a = h2.get_info(); + + // allow get_info to fire for h2. + t_handle.tick(2); + + // post-conditions: + // 1) track 1 is done & dropped (commands fail). + // 2) track 2 is playing. + assert!(h1a.await.is_err()); + assert_eq!(h2a.await.unwrap().playing, PlayMode::Play); + } +} diff --git a/src/tracks/ready.rs b/src/tracks/ready.rs new file mode 100644 index 0000000..0114536 --- /dev/null +++ b/src/tracks/ready.rs @@ -0,0 +1,21 @@ +/// Whether this track has been made live, is being processed, or is +/// currently uninitialised. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub enum ReadyState { + /// This track is still a lazy [`Compose`] object, and hasn't been made playable. + /// + /// [`Compose`]: crate::input::Compose + Uninitialised, + + /// The mixer is currently creating and parsing this track's bytestream. + Preparing, + + /// This track is fully initialised and usable. + Playable, +} + +impl Default for ReadyState { + fn default() -> Self { + Self::Uninitialised + } +} diff --git a/src/tracks/state.rs b/src/tracks/state.rs index 3be47fb..229a5eb 100644 --- a/src/tracks/state.rs +++ b/src/tracks/state.rs @@ -5,21 +5,29 @@ use super::*; /// /// [`Track`]: Track /// [`TrackHandle::get_info`]: TrackHandle::get_info -#[derive(Copy, Clone, Debug, Default, PartialEq)] +#[derive(Clone, Debug, Default, PartialEq)] pub struct TrackState { /// Play status (e.g., active, paused, stopped) of this track. pub playing: PlayMode, + /// Current volume of this track. pub volume: f32, + /// Current playback position in the source. /// /// This is altered by loops and seeks, and represents this track's /// position in its underlying input stream. pub position: Duration, + /// Total playback time, increasing monotonically. pub play_time: Duration, + /// Remaining loops on this track. pub loops: LoopState, + + /// Whether this track has been made live, is being processed, or is + /// currently uninitialised. + pub ready: ReadyState, } impl TrackState { @@ -28,3 +36,35 @@ impl TrackState { self.play_time += TIMESTEP_LENGTH; } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + constants::test_data::YTDL_TARGET, + driver::Driver, + input::YoutubeDl, + tracks::Track, + Config, + }; + use reqwest::Client; + + #[tokio::test] + #[ntest::timeout(10_000)] + async fn times_unchanged_while_not_ready() { + let (t_handle, config) = Config::test_cfg(true); + let mut driver = Driver::new(config.clone()); + + let file = YoutubeDl::new(Client::new(), YTDL_TARGET.into()); + let handle = driver.play(Track::from(file)); + + let state = t_handle + .ready_track(&handle, Some(Duration::from_millis(5))) + .await; + + // As state is `play`, the instant we ready we'll have playout. + // Naturally, fetching a ytdl request takes far longer than this. + assert_eq!(state.position, Duration::from_millis(20)); + assert_eq!(state.play_time, Duration::from_millis(20)); + } +} diff --git a/src/tracks/view.rs b/src/tracks/view.rs new file mode 100644 index 0000000..dd4d3a6 --- /dev/null +++ b/src/tracks/view.rs @@ -0,0 +1,31 @@ +use super::*; +use crate::input::Metadata; +use std::time::Duration; + +/// Live track and input state exposed during [`TrackHandle::action`]. +/// +/// [`TrackHandle::action`]: super::[`TrackHandle::action`] +#[non_exhaustive] +pub struct View<'a> { + /// The current position within this track. + pub position: &'a Duration, + + /// The total time a track has been played for. + pub play_time: &'a Duration, + + /// The current mixing volume of this track. + pub volume: &'a mut f32, + + /// In-stream metadata for this track, if it is fully readied. + pub meta: Option>, + + /// The current play status of this track. + pub playing: &'a mut PlayMode, + + /// Whether this track has been made live, is being processed, or is + /// currently uninitialised. + pub ready: ReadyState, + + /// The number of remaning loops on this track. + pub loops: &'a mut LoopState, +}