s&box Package Code Search
Search C# source code, UI razor templates, shaders, and configs across s&box packages.
link Raw Facepunch API Link: https://public.facepunch.com/sbox/code/search/1/?ident=rexi.bimp&take=20
Showing code results for query:
*
(124 total matches found)
Game
library
using System;
using Sandbox.UI;
namespace Bimp;
/// <summary>
/// The button for an input action, drawn the way the player's device shows it: a key cap on keyboard, the pad's own
/// button (Xbox, PlayStation, Nintendo...) on a controller. It follows them live as they switch device. Sized to the
/// text around it (1.25em tall, as wide as the glyph).
/// </summary>
[StyleSheet.Inline( "bimp-inputglyph", Styles )]
public sealed class InputGlyph : Panel
{
const string Styles = """
.inputglyph { height: 1.25em; aspect-ratio: 1; flex-shrink: 0; background-size: contain; background-repeat: no-repeat; background-position: center; }
""";
/// <summary> The input action to show the button of, e.g. "use". </summary>
public string Action { get; set; }
/// <summary> An analog input to show instead of an action, by <see cref="InputAnalog"/> name, e.g. "LeftStickY". </summary>
public string Analog { get; set; }
/// <summary> A keyboard key to show instead of an action, e.g. "escape" (keyboard glyph whatever the device). </summary>
public string Key { get; set; }
/// <summary> Glyph resolution. Medium (128px) stays sharp on the big world screens. </summary>
public InputGlyphSize Size { get; set; } = InputGlyphSize.Medium;
Texture shown;
public InputGlyph()
{
AddClass( "inputglyph" );
}
public override void Tick()
{
base.Tick();
// cheap, and the device can change any frame (the engine's own advice for glyphs)
var texture = Glyph();
if ( texture == shown ) return;
shown = texture;
Style.SetBackgroundImage( texture );
Style.AspectRatio = texture is { Height: > 0 } ? texture.Width / (float)texture.Height : 1;
}
Texture Glyph()
{
if ( !string.IsNullOrWhiteSpace( Key ) ) return Input.Keyboard.GetGlyph( Key, Size );
if ( Enum.TryParse<InputAnalog>( Analog, true, out var analog ) ) return Input.GetGlyph( analog, Size );
if ( !string.IsNullOrWhiteSpace( Action ) ) return Input.GetGlyph( Action, Size, false );
return null;
}
}
Game
library
using System.Threading;
using Bimp.Resolver.Live;
namespace Bimp.Resolver.Extractors;
/// <summary>
/// Any .mpd link: a finished (static) DASH manifest plays as a seekable video, remuxed from a start time like a finished
/// HLS playlist (see <see cref="MpdPlaylists"/>, which turns the manifest into playlists <see cref="HlsReader"/> reads).
/// </summary>
public sealed class DashExtractor : IExtractor
{
public string Key => "dash";
public bool CanHandle( Uri url ) => MpdPlaylists.IsMpd( url );
public async Task<MediaInfo> GetInfoAsync( Uri url, bool audioOnly, CancellationToken ct )
{
var mpd = await MpdPlaylists.LoadAsync( url, ct );
var name = Uri.UnescapeDataString( System.IO.Path.GetFileNameWithoutExtension( url.AbsolutePath ) );
return new MediaInfo
{
Extractor = Key,
Id = url.ToString(),
Title = !string.IsNullOrWhiteSpace( mpd.Title ) ? mpd.Title : name is "manifest" or "index" or "stream" or "" ? url.Host : $"{url.Host} - {name}",
Duration = (float)mpd.Duration,
AudioOnly = audioOnly || mpd.Heights.Count == 0,
Merged = true,
Qualities = audioOnly || mpd.Heights.Count < 2 ? new List<int>() : mpd.Heights,
};
}
public Task<StreamPlan> GetStreamsAsync( string id, StreamRequest request, CancellationToken ct )
=> Task.FromResult( new StreamPlan { Kind = StreamKind.Hls, DirectUrl = id } );
}
Game
library
using System.Buffers.Binary;
using System.IO;
using System.Text;
namespace Bimp.Resolver.Live;
/// <summary> One video access unit (timestamps in 90 kHz units). </summary>
public sealed class VideoSample
{
public long Dts;
public long Pts;
public bool Key;
/// <summary> H.264: AVCC (NAL units with 4 byte lengths). AV1: a temporal unit of sized OBUs. </summary>
public byte[] Data;
}
/// <summary> A video track's codec and decoder configuration: what its MP4 sample entry says. </summary>
public sealed class VideoFormat
{
public string Codec { get; private init; }
public int Width { get; private init; }
public int Height { get; private init; }
/// <summary> The sample entry's four character code and its configuration box. </summary>
internal string Fourcc { get; private init; }
internal string ConfigBox { get; private init; }
internal byte[] Config { get; private init; }
/// <summary>
/// The engine's H.264 decoder (Media Foundation) holds back the last 22 frames of a file; its AV1 decoder shows
/// every frame (see <see cref="Media.Av1"/>).
/// </summary>
public bool HoldsBackFrames => Fourcc == "avc1";
public bool IsH264 => Fourcc == "avc1";
public static VideoFormat H264( byte[] sps, byte[] pps )
{
var info = Live.H264.ParseSps( sps );
return new VideoFormat { Codec = "H.264", Width = info.Width, Height = info.Height, Fourcc = "avc1", ConfigBox = "avcC", Config = Live.H264.AvcConfig( sps, pps ) };
}
public static VideoFormat Av1( Media.Av1.SequenceHeader seq )
=> new() { Codec = "AV1", Width = seq.Width, Height = seq.Height, Fourcc = "av01", ConfigBox = "av1C", Config = Media.Av1.ConfigRecord( seq ) };
public override string ToString() => $"{Codec} {Width}x{Height}";
}
/// <summary> One raw AAC frame (no ADTS header), 1024 samples. Pts in 90 kHz units. </summary>
public sealed class AudioSample
{
public long Pts;
public byte[] Data;
}
/// <summary> An AAC track's decoder configuration. </summary>
public sealed class AacConfig
{
/// <summary> AudioSpecificConfig. </summary>
public byte[] Asc;
public int SampleRate;
public int Channels;
static readonly int[] Rates = { 96000, 88200, 64000, 48000, 44100, 32000, 24000, 22050, 16000, 12000, 11025, 8000, 7350 };
public static int RateIndex( int rate ) => Array.IndexOf( Rates, rate );
public static int RateOf( int index ) => index >= 0 && index < Rates.Length ? Rates[index] : 0;
/// <summary> Build from ADTS header fields. </summary>
public static AacConfig FromAdts( int profile, int rateIndex, int channelConfig )
{
var objectType = profile + 1;
var asc = new byte[]
{
(byte)((objectType << 3) | (rateIndex >> 1)),
(byte)(((rateIndex & 1) << 7) | (channelConfig << 3)),
};
return new AacConfig { Asc = asc, SampleRate = RateOf( rateIndex ), Channels = channelConfig == 7 ? 8 : channelConfig };
}
/// <summary> Parse an AudioSpecificConfig (from SDP "config=" or an esds). </summary>
public static AacConfig FromAsc( byte[] asc )
{
if ( asc is null || asc.Length < 2 ) return null;
var v = (asc[0] << 8) | asc[1];
var rateIndex = (v >> 7) & 0xF;
var channelConfig = (v >> 3) & 0xF;
return new AacConfig { Asc = asc, SampleRate = RateOf( rateIndex ), Channels = channelConfig == 7 ? 8 : Math.Max( 1, channelConfig ) };
}
public bool SameAs( AacConfig o ) => o is not null && Asc.AsSpan().SequenceEqual( o.Asc );
/// <summary>
/// The rate the mp4a sample entry declares. The engine's AAC decoder (Media Foundation) won't open audio below
/// 32 kHz - measured 2026-10-01: 16, 22.05 and 24 kHz AAC-LC play nothing, 32/44.1/48 kHz play - and an HE-AAC
/// stream's core (all the ASC says when the SBR is implicit, as from ADTS) is half its real rate. Declaring twice
/// the core is how Media Foundation is told the stream is HE-AAC: it then opens it at the doubled rate.
/// </summary>
public int OutputRate => SampleRate is > 0 and < 32000 ? SampleRate * 2 : SampleRate;
}
/// <summary>
/// Writes a finished, moov-first MP4 (H.264 and/or AAC) - the only kind of file the engine opens instantly
/// (it walks every fragment of a fragmented MP4 on open, and needs the index before the data).
/// The file's timeline starts at <see cref="Write"/>'s <c>zero</c>.
/// </summary>
public static class Mp4Writer
{
const uint MovieTimescale = 90000;
public static void Write( Stream output, long zero,
List<VideoSample> video, byte[] sps, byte[] pps,
List<AudioSample> audio, AacConfig aac )
=> Write( output, zero, video, sps is null || pps is null ? null : VideoFormat.H264( sps, pps ), audio, aac );
public static void Write( Stream output, long zero,
List<VideoSample> video, VideoFormat format,
List<AudioSample> audio, AacConfig aac )
{
var hasVideo = video is { Count: > 0 } && format is not null;
var hasAudio = audio is { Count: > 0 } && aac is not null && aac.SampleRate > 0;
var ftyp = Box( "ftyp", Concat( Ascii( "isom" ), U32( 512 ), Ascii( "isom" ), Ascii( "iso2" ), Ascii( format?.Fourcc ?? "avc1" ), Ascii( "mp41" ) ) );
// sample data: all video, then all audio (one chunk each - it's a local file, no need to interleave)
var videoData = hasVideo ? video.Select( s => s.Data ).ToList() : new List<byte[]>();
var audioData = hasAudio ? audio.Select( s => s.Data ).ToList() : new List<byte[]>();
long videoBytes = videoData.Sum( d => (long)d.Length );
// moov has a fixed size whatever the offsets are, so build it once to measure, then for real
byte[] Moov( uint videoOffset, uint audioOffset )
{
var traks = new List<byte[]>();
uint id = 1;
if ( hasVideo ) traks.Add( VideoTrak( id++, zero, video, format, videoOffset ) );
if ( hasAudio ) traks.Add( AudioTrak( id++, zero, audio, aac, audioOffset ) );
var duration = Math.Max( hasVideo ? VideoDuration( video, zero ) : 0, hasAudio ? AudioMovieDuration( audio, aac, zero ) : 0 );
return Box( "moov", Concat( new[] { Mvhd( duration, id ) }.Concat( traks ).ToArray() ) );
}
var moovSize = Moov( 0, 0 ).Length;
var dataStart = (uint)(ftyp.Length + moovSize + 8);
var moov = Moov( dataStart, (uint)(dataStart + videoBytes) );
var mdatSize = 8 + videoBytes + audioData.Sum( d => (long)d.Length );
output.Write( ftyp );
output.Write( moov );
output.Write( U32( (uint)mdatSize ) );
output.Write( Ascii( "mdat" ) );
foreach ( var d in videoData ) output.Write( d );
foreach ( var d in audioData ) output.Write( d );
}
static long VideoDuration( List<VideoSample> v, long zero )
{
var last = v[^1];
var frame = v.Count > 1 ? Math.Max( 1, last.Dts - v[^2].Dts ) : 3000;
return last.Dts + frame - zero;
}
static long AudioMovieDuration( List<AudioSample> a, AacConfig aac, long zero )
=> Math.Max( 0, a[0].Pts - zero ) + (long)a.Count * 1024 * MovieTimescale / aac.SampleRate;
static byte[] VideoTrak( uint id, long zero, List<VideoSample> v, VideoFormat info, uint offset )
{
// durations between decode times; the last frame lasts as long as the one before it
var durations = new List<uint>();
for ( int i = 0; i < v.Count; i++ )
{
var d = i + 1 < v.Count ? v[i + 1].Dts - v[i].Dts : (v.Count > 1 ? v[i].Dts - v[i - 1].Dts : 3000);
durations.Add( (uint)Math.Clamp( d, 1, 90000 ) );
}
var mediaDuration = durations.Sum( d => (long)d );
var offsets = v.Select( s => s.Pts - s.Dts ).ToList();
var hasCtts = offsets.Any( o => o != 0 );
var stbl = new List<byte[]>
{
Box( "stsd", Concat( U32( 0 ), U32( 1 ), VisualEntry( info ) ) ),
Stts( durations ),
};
if ( hasCtts ) stbl.Add( Ctts( offsets ) );
stbl.Add( Stss( v ) );
stbl.Add( Stsc( v.Count ) );
stbl.Add( Stsz( v.Select( s => s.Data.Length ) ) );
stbl.Add( Box( "stco", Concat( U32( 0 ), U32( 1 ), U32( offset ) ) ) );
// the first frame shows at its presentation time; start the track there
var elst = Elst( Math.Max( 0, v[0].Dts - zero ), mediaDuration, Math.Max( 0, offsets[0] ) );
return Box( "trak", Concat(
Tkhd( id, mediaDuration + Math.Max( 0, v[0].Dts - zero ), false, info.Width, info.Height ),
Box( "edts", elst ),
Box( "mdia", Concat(
Mdhd( 90000, mediaDuration ),
Hdlr( "vide" ),
Box( "minf", Concat(
FullBox( "vmhd", 0, 1, new byte[8] ),
Dinf(),
Box( "stbl", Concat( stbl.ToArray() ) ) ) ) ) ) ) );
}
static byte[] AudioTrak( uint id, long zero, List<AudioSample> a, AacConfig aac, uint offset )
{
var mediaDuration = (long)a.Count * 1024;
var startDelay = Math.Max( 0, a[0].Pts - zero ); // movie timescale
var stbl = Concat(
Box( "stsd", Concat( U32( 0 ), U32( 1 ), Mp4a( aac ) ) ),
Stts( Enumerable.Repeat( 1024u, a.Count ).ToList() ),
Stsc( a.Count ),
Stsz( a.Select( s => s.Data.Length ) ),
Box( "stco", Concat( U32( 0 ), U32( 1 ), U32( offset ) ) ) );
var movieDuration = mediaDuration * MovieTimescale / aac.SampleRate;
return Box( "trak", Concat(
Tkhd( id, startDelay + movieDuration, true, 0, 0 ),
Box( "edts", Elst( startDelay, movieDuration, 0 ) ),
Box( "mdia", Concat(
Mdhd( (uint)aac.SampleRate, mediaDuration ),
Hdlr( "soun" ),
Box( "minf", Concat(
FullBox( "smhd", 0, 0, new byte[4] ),
Dinf(),
Box( "stbl", stbl ) ) ) ) ) ) );
}
/// <summary> An edit list: an optional empty edit (delay), then the media from mediaTime. </summary>
static byte[] Elst( long delay, long movieDuration, long mediaTime )
{
var entries = new List<byte[]>();
if ( delay > 0 ) entries.Add( Concat( U32( (uint)delay ), U32( 0xFFFFFFFF ), U32( 0x00010000 ) ) );
entries.Add( Concat( U32( (uint)movieDuration ), U32( (uint)mediaTime ), U32( 0x00010000 ) ) );
return FullBox( "elst", 0, 0, Concat( new[] { U32( (uint)entries.Count ) }.Concat( entries ).ToArray() ) );
}
static byte[] Mvhd( long duration, uint nextTrack )
{
return FullBox( "mvhd", 0, 0, Concat(
U32( 0 ), U32( 0 ), U32( MovieTimescale ), U32( (uint)duration ),
U32( 0x00010000 ), U16( 0x0100 ), new byte[10],
Matrix(), new byte[24], U32( nextTrack ) ) );
}
static byte[] Tkhd( uint id, long duration, bool audio, int width, int height )
{
return FullBox( "tkhd", 0, 3, Concat(
U32( 0 ), U32( 0 ), U32( id ), U32( 0 ), U32( (uint)duration ),
new byte[8], U16( 0 ), U16( 0 ), U16( (ushort)(audio ? 0x0100 : 0) ), U16( 0 ),
Matrix(), U32( (uint)width << 16 ), U32( (uint)height << 16 ) ) );
}
static byte[] Mdhd( uint timescale, long duration )
=> FullBox( "mdhd", 0, 0, Concat( U32( 0 ), U32( 0 ), U32( timescale ), U32( (uint)duration ), U16( 0x55C4 ), U16( 0 ) ) );
static byte[] Hdlr( string type )
=> FullBox( "hdlr", 0, 0, Concat( U32( 0 ), Ascii( type ), new byte[12], Encoding.ASCII.GetBytes( "bimp\0" ) ) );
static byte[] Dinf() => Box( "dinf", FullBox( "dref", 0, 0, Concat( U32( 1 ), FullBox( "url ", 0, 1, Array.Empty<byte>() ) ) ) );
/// <summary> avc1 + avcC, or av01 + av1C. </summary>
static byte[] VisualEntry( VideoFormat f )
{
return Box( f.Fourcc, Concat(
new byte[6], U16( 1 ), // reserved, data_reference_index
new byte[16], // pre_defined, reserved, pre_defined[3]
U16( (ushort)f.Width ), U16( (ushort)f.Height ),
U32( 0x00480000 ), U32( 0x00480000 ), U32( 0 ), U16( 1 ),
new byte[32], // compressorname
U16( 0x0018 ), U16( 0xFFFF ),
Box( f.ConfigBox, f.Config ) ) );
}
static byte[] Mp4a( AacConfig aac )
{
// ES_Descriptor > DecoderConfigDescriptor (AAC, audio stream) > DecoderSpecificInfo (the ASC), SLConfig
var dsi = Descriptor( 5, aac.Asc );
var dcd = Descriptor( 4, Concat( new byte[] { 0x40, 0x15, 0, 0, 0 }, U32( 0 ), U32( 0 ), dsi ) );
var es = Descriptor( 3, Concat( U16( 1 ), new byte[] { 0 }, dcd, Descriptor( 6, new byte[] { 2 } ) ) );
// channelcount must match the ASC - the engine trusts this field (see Mp4ChannelFix)
return Box( "mp4a", Concat(
new byte[6], U16( 1 ), new byte[8],
U16( (ushort)aac.Channels ), U16( 16 ), U16( 0 ), U16( 0 ),
U32( (uint)Math.Min( aac.OutputRate, 65535 ) << 16 ),
FullBox( "esds", 0, 0, es ) ) );
}
static byte[] Descriptor( byte tag, byte[] body )
{
// 4 byte length form, like most muxers write it
var len = body.Length;
return Concat( new byte[] { tag, (byte)(0x80 | (len >> 21) & 0x7F), (byte)(0x80 | (len >> 14) & 0x7F), (byte)(0x80 | (len >> 7) & 0x7F), (byte)(len & 0x7F) }, body );
}
static byte[] Stts( List<uint> durations )
{
var runs = new List<(uint count, uint delta)>();
foreach ( var d in durations )
{
if ( runs.Count > 0 && runs[^1].delta == d ) runs[^1] = (runs[^1].count + 1, d);
else runs.Add( (1, d) );
}
return FullBox( "stts", 0, 0, Concat( new[] { U32( (uint)runs.Count ) }.Concat( runs.Select( r => Concat( U32( r.count ), U32( r.delta ) ) ) ).ToArray() ) );
}
static byte[] Ctts( List<long> offsets )
{
// version 0 (unsigned) like normal muxers write it - B-frame offsets are never negative when decode time is the
// earliest presentation; version 1 (signed) only if we really need it
var signed = offsets.Any( o => o < 0 );
var runs = new List<(uint count, long offset)>();
foreach ( var o in offsets )
{
if ( runs.Count > 0 && runs[^1].offset == o ) runs[^1] = (runs[^1].count + 1, o);
else runs.Add( (1, o) );
}
return FullBox( "ctts", (byte)(signed ? 1 : 0), 0, Concat( new[] { U32( (uint)runs.Count ) }.Concat( runs.Select( r => Concat( U32( r.count ), U32( unchecked((uint)(int)r.offset) ) ) ) ).ToArray() ) );
}
static byte[] Stss( List<VideoSample> v )
{
var keys = v.Select( ( s, i ) => (s, i) ).Where( x => x.s.Key ).Select( x => U32( (uint)x.i + 1 ) ).ToList();
return FullBox( "stss", 0, 0, Concat( new[] { U32( (uint)keys.Count ) }.Concat( keys ).ToArray() ) );
}
static byte[] Stsc( int samples ) => FullBox( "stsc", 0, 0, Concat( U32( 1 ), U32( 1 ), U32( (uint)samples ), U32( 1 ) ) );
static byte[] Stsz( IEnumerable<int> sizes )
{
var list = sizes.ToList();
return FullBox( "stsz", 0, 0, Concat( new[] { U32( 0 ), U32( (uint)list.Count ) }.Concat( list.Select( s => U32( (uint)s ) ) ).ToArray() ) );
}
static byte[] Matrix() => Concat( U32( 0x00010000 ), U32( 0 ), U32( 0 ), U32( 0 ), U32( 0x00010000 ), U32( 0 ), U32( 0 ), U32( 0 ), U32( 0x40000000 ) );
static byte[] Box( string type, byte[] body ) => Concat( U32( (uint)(8 + body.Length) ), Ascii( type ), body );
static byte[] FullBox( string type, byte version, uint flags, byte[] body )
=> Box( type, Concat( new[] { version, (byte)(flags >> 16), (byte)(flags >> 8), (byte)flags }, body ) );
static byte[] Ascii( string s ) => Encoding.ASCII.GetBytes( s );
static byte[] U32( uint v ) { var b = new byte[4]; BinaryPrimitives.WriteUInt32BigEndian( b, v ); return b; }
static byte[] U16( ushort v ) { var b = new byte[2]; BinaryPrimitives.WriteUInt16BigEndian( b, v ); return b; }
static byte[] Concat( params byte[][] parts ) => Media.Ebml.Concat( parts );
}
Game
library
using System.Buffers.Binary;
using System.Text;
namespace Bimp.Resolver.Media;
/// <summary>
/// Reads fragmented MP4 (the CMAF / DASH / HLS fMP4 layout): the tracks in an init segment's moov, and the samples in
/// moof + mdat fragments. Shared by YouTube's AV1 files (<see cref="WebmSource.ProbeMp4Async"/>) and fMP4 HLS.
/// </summary>
public static class Fmp4Reader
{
/// <summary> One track of an init segment. </summary>
public sealed class Track
{
public uint Id;
/// <summary> "vide" or "soun". </summary>
public string Handler;
public long Timescale;
public double DurationSeconds;
/// <summary> The sample entry's four character code: avc1, avc3, hvc1, hev1, av01, mp4a, Opus... </summary>
public string Codec;
/// <summary> avcC / av1C body, or the AAC AudioSpecificConfig from the esds. </summary>
public byte[] Config;
public int Width, Height;
public int Channels, SampleRate;
// trex defaults, for fragments that don't set their own
public uint DefaultDuration, DefaultSize, DefaultFlags;
public bool IsVideo => Handler == "vide";
public bool IsAudio => Handler == "soun";
}
/// <summary> One sample of a fragment: where its data is in the buffer, and its times in the track's timescale. </summary>
public readonly record struct Sample( uint Track, long DecodeTime, int CompositionOffset, uint Duration, bool Key, int Offset, int Size );
/// <summary> The tracks of the moov in d[start..end] (an init segment, or the head of a DASH file). </summary>
public static List<Track> ReadTracks( byte[] d, int start, int end )
{
var moov = Find( d, start, end, "moov" ) ?? throw new MergeException( "no moov in the mp4" );
var tracks = new List<Track>();
var trex = new Dictionary<uint, (uint duration, uint size, uint flags)>();
foreach ( var (at, size, type, h) in Children( d, moov.at + moov.h, moov.at + moov.size ) )
{
if ( type == "trak" && ReadTrak( d, at, at + size ) is { } t ) tracks.Add( t );
else if ( type == "mvex" )
{
foreach ( var (x, xs, xt, xh) in Children( d, at + h, at + size ) )
{
if ( xt != "trex" ) continue;
// full box: version/flags, track_ID, description index, duration, size, flags
var p = x + xh + 4;
trex[BinaryPrimitives.ReadUInt32BigEndian( d.AsSpan( p ) )] = (
BinaryPrimitives.ReadUInt32BigEndian( d.AsSpan( p + 8 ) ),
BinaryPrimitives.ReadUInt32BigEndian( d.AsSpan( p + 12 ) ),
BinaryPrimitives.ReadUInt32BigEndian( d.AsSpan( p + 16 ) ));
}
}
}
foreach ( var t in tracks )
{
if ( trex.TryGetValue( t.Id, out var x ) ) (t.DefaultDuration, t.DefaultSize, t.DefaultFlags) = x;
}
return tracks;
}
static Track ReadTrak( byte[] d, int trak, int end )
{
var t = new Track();
if ( Path( d, trak, end, "tkhd" ) is { } tkhd )
{
var p = tkhd.at + tkhd.h;
t.Id = BinaryPrimitives.ReadUInt32BigEndian( d.AsSpan( p + (d[p] == 1 ? 20 : 12) ) );
}
var mdhd = Path( d, trak, end, "mdia", "mdhd" );
if ( mdhd is null ) return null;
var mp = mdhd.Value.at + mdhd.Value.h;
long duration;
if ( d[mp] == 1 )
{
t.Timescale = BinaryPrimitives.ReadUInt32BigEndian( d.AsSpan( mp + 20 ) );
duration = (long)BinaryPrimitives.ReadUInt64BigEndian( d.AsSpan( mp + 24 ) );
}
else
{
t.Timescale = BinaryPrimitives.ReadUInt32BigEndian( d.AsSpan( mp + 12 ) );
duration = BinaryPrimitives.ReadUInt32BigEndian( d.AsSpan( mp + 16 ) );
}
if ( t.Timescale <= 0 ) return null;
t.DurationSeconds = duration / (double)t.Timescale;
if ( Path( d, trak, end, "mdia", "hdlr" ) is { } hdlr ) t.Handler = Encoding.ASCII.GetString( d, hdlr.at + hdlr.h + 8, 4 );
var stsd = Path( d, trak, end, "mdia", "minf", "stbl", "stsd" );
if ( stsd is null ) return t;
var entry = stsd.Value.at + stsd.Value.h + 8; // version/flags, entry count
var (entrySize, codec, entryHead) = Box( d, entry, stsd.Value.at + stsd.Value.size - entry );
t.Codec = codec;
var entryEnd = entry + (int)entrySize;
if ( t.IsVideo )
{
// VisualSampleEntry: 6 reserved, data reference index, 16 bytes of pre-defined/reserved, then width and height
t.Width = BinaryPrimitives.ReadUInt16BigEndian( d.AsSpan( entry + entryHead + 24 ) );
t.Height = BinaryPrimitives.ReadUInt16BigEndian( d.AsSpan( entry + entryHead + 26 ) );
var configType = codec switch { "avc1" or "avc3" => "avcC", "av01" => "av1C", "hvc1" or "hev1" => "hvcC", _ => null };
if ( configType is not null && Find( d, entry + entryHead + 78, entryEnd, configType ) is { } c )
t.Config = d.AsSpan( c.at + c.h, c.size - c.h ).ToArray();
}
else if ( t.IsAudio )
{
// AudioSampleEntry: 6 reserved, data reference index, 8 reserved, channels, sample size, 4 reserved, rate 16.16
t.Channels = BinaryPrimitives.ReadUInt16BigEndian( d.AsSpan( entry + entryHead + 16 ) );
t.SampleRate = BinaryPrimitives.ReadUInt16BigEndian( d.AsSpan( entry + entryHead + 24 ) );
if ( codec == "mp4a" && Find( d, entry + entryHead + 28, entryEnd, "esds" ) is { } esds )
t.Config = AudioSpecificConfig( d.AsSpan( esds.at + esds.h + 4, esds.size - esds.h - 4 ) );
}
return t;
}
/// <summary> The DecoderSpecificInfo (tag 5) inside an esds's descriptors: an AAC AudioSpecificConfig. </summary>
static byte[] AudioSpecificConfig( ReadOnlySpan<byte> d )
{
var i = 0;
while ( i + 2 <= d.Length )
{
var tag = d[i++];
var length = 0;
for ( int k = 0; k < 4 && i < d.Length; k++ )
{
var b = d[i++];
length = (length << 7) | (b & 0x7F);
if ( (b & 0x80) == 0 ) break;
}
if ( tag == 5 ) return d.Slice( i, Math.Min( length, d.Length - i ) ).ToArray();
if ( tag == 3 )
{
// ES_Descriptor: ES_ID, flags, then optional fields - its children follow
var flags = d[i + 2];
i += 3;
if ( (flags & 0x80) != 0 ) i += 2;
if ( (flags & 0x40) != 0 ) i += 1 + d[i];
if ( (flags & 0x20) != 0 ) i += 2;
}
else if ( tag == 4 ) i += 13; // DecoderConfigDescriptor: object type .. average bitrate, then children
else i += length;
}
return null;
}
/// <summary>
/// The samples of every fragment (moof + mdat) in d[start..end], in order. <paramref name="fileOffset"/> is where
/// d[0] is in the file (for base data offsets); offsets in the result are into d. <paramref name="partial"/>: the
/// last fragment may be cut off - its samples stop where the data does.
/// </summary>
public static List<Sample> ReadSamples( byte[] d, int start, int end, IReadOnlyList<Track> tracks, long fileOffset = 0, bool partial = false )
{
var samples = new List<Sample>();
foreach ( var (moof, moofSize, type, moofHead) in Children( d, start, end ) )
{
if ( type != "moof" ) continue;
foreach ( var (traf, trafSize, trafType, trafHead) in Children( d, moof + moofHead, moof + moofSize ) )
{
if ( trafType != "traf" ) continue;
long baseOffset = moof, decodeTime = 0;
uint trackId = 0, duration = 0, size = 0, flags = 0;
Track track = null;
foreach ( var (box, _, boxType, boxHead) in Children( d, traf + trafHead, traf + trafSize ) )
{
var p = box + boxHead;
if ( boxType == "tfhd" )
{
var f = BinaryPrimitives.ReadUInt32BigEndian( d.AsSpan( p ) ) & 0xFFFFFF;
trackId = BinaryPrimitives.ReadUInt32BigEndian( d.AsSpan( p + 4 ) );
track = tracks.FirstOrDefault( t => t.Id == trackId ) ?? (tracks.Count == 1 ? tracks[0] : null);
duration = track?.DefaultDuration ?? 0;
size = track?.DefaultSize ?? 0;
flags = track?.DefaultFlags ?? 0;
p += 8; // version/flags, track_ID
if ( (f & 0x01) != 0 ) { baseOffset = (long)BinaryPrimitives.ReadUInt64BigEndian( d.AsSpan( p ) ) - fileOffset; p += 8; }
if ( (f & 0x02) != 0 ) p += 4;
if ( (f & 0x08) != 0 ) { duration = BinaryPrimitives.ReadUInt32BigEndian( d.AsSpan( p ) ); p += 4; }
if ( (f & 0x10) != 0 ) { size = BinaryPrimitives.ReadUInt32BigEndian( d.AsSpan( p ) ); p += 4; }
if ( (f & 0x20) != 0 ) flags = BinaryPrimitives.ReadUInt32BigEndian( d.AsSpan( p ) );
}
else if ( boxType == "tfdt" )
{
decodeTime = d[p] == 1 ? (long)BinaryPrimitives.ReadUInt64BigEndian( d.AsSpan( p + 4 ) ) : BinaryPrimitives.ReadUInt32BigEndian( d.AsSpan( p + 4 ) );
}
else if ( boxType == "trun" )
{
var version = d[p];
var f = BinaryPrimitives.ReadUInt32BigEndian( d.AsSpan( p ) ) & 0xFFFFFF;
var count = BinaryPrimitives.ReadUInt32BigEndian( d.AsSpan( p + 4 ) );
p += 8;
long data = baseOffset;
if ( (f & 0x01) != 0 ) { data = baseOffset + BinaryPrimitives.ReadInt32BigEndian( d.AsSpan( p ) ); p += 4; }
uint? firstFlags = null;
if ( (f & 0x04) != 0 ) { firstFlags = BinaryPrimitives.ReadUInt32BigEndian( d.AsSpan( p ) ); p += 4; }
for ( uint i = 0; i < count; i++ )
{
uint sDuration = duration, sSize = size, sFlags = i == 0 && firstFlags is { } ff ? ff : flags;
var offset = 0;
if ( (f & 0x100) != 0 ) { sDuration = BinaryPrimitives.ReadUInt32BigEndian( d.AsSpan( p ) ); p += 4; }
if ( (f & 0x200) != 0 ) { sSize = BinaryPrimitives.ReadUInt32BigEndian( d.AsSpan( p ) ); p += 4; }
if ( (f & 0x400) != 0 ) { sFlags = BinaryPrimitives.ReadUInt32BigEndian( d.AsSpan( p ) ); p += 4; }
if ( (f & 0x800) != 0 ) { offset = BinaryPrimitives.ReadInt32BigEndian( d.AsSpan( p ) ); if ( version == 0 ) offset = (int)Math.Min( (uint)offset, int.MaxValue ); p += 4; }
if ( partial && data >= 0 && data + sSize > Math.Min( end, d.Length ) ) return samples;
if ( data < 0 || data + sSize > d.Length ) throw new MergeException( "mp4 sample outside the fragment" );
// audio samples are all sync samples, whatever the flags say
var key = (sFlags & 0x10000) == 0 || (track?.IsAudio ?? false);
samples.Add( new Sample( track?.Id ?? trackId, decodeTime, offset, sDuration, key, (int)data, (int)sSize ) );
data += sSize;
decodeTime += sDuration;
}
}
}
}
}
return samples;
}
public static (long size, string type, int headerLength) Box( byte[] d, int at, long remaining )
{
if ( at + 8 > d.Length ) return (0, null, 0);
long size = BinaryPrimitives.ReadUInt32BigEndian( d.AsSpan( at ) );
var type = Encoding.ASCII.GetString( d, at + 4, 4 );
var h = 8;
if ( size == 1 )
{
if ( at + 16 > d.Length ) return (0, null, 0);
size = (long)BinaryPrimitives.ReadUInt64BigEndian( d.AsSpan( at + 8 ) );
h = 16;
}
else if ( size == 0 ) size = remaining;
return (size < h ? 0 : size, type, h);
}
public static IEnumerable<(int at, int size, string type, int head)> Children( byte[] d, int start, int end )
{
for ( var at = start; at + 8 <= end; )
{
var (size, type, h) = Box( d, at, end - at );
if ( size <= 0 || at + size > end ) yield break;
yield return (at, (int)size, type, h);
at += (int)size;
}
}
public static (int at, int size, int h)? Find( byte[] d, int start, int end, string type )
{
foreach ( var c in Children( d, start, end ) )
if ( c.type == type ) return (c.at, c.size, c.head);
return null;
}
/// <summary> A box nested under a box at <paramref name="at"/>, by the types of its ancestors below it. </summary>
public static (int at, int size, int h)? Path( byte[] d, int at, int end, params string[] types )
{
var head = Box( d, at, end - at ).headerLength;
(int at, int size, int h)? found = null;
int s = at + head, e = end;
foreach ( var t in types )
{
found = Find( d, s, e, t );
if ( found is not { } f ) return null;
s = f.at + f.h;
e = f.at + f.size;
}
return found;
}
}
Game
library
using System.Threading;
namespace Bimp.Resolver;
/// <summary>
/// A user facing resolve failure ("This video is private.").
/// </summary>
public class ResolveException : Exception
{
public ResolveException( string message ) : base( message ) { }
}
/// <summary>
/// What the host learns about a piece of media - shared with everyone through the synced queue item.
/// Contains no stream urls: those are tied to the IP that asked for them, so every client gets its own.
/// </summary>
public sealed class MediaInfo
{
/// <summary> The extractor that understood the url (<see cref="IExtractor.Key"/>). </summary>
public string Extractor { get; set; }
/// <summary> Extractor specific id (YouTube video id, SoundCloud url...). </summary>
public string Id { get; set; }
public string Title { get; set; }
/// <summary> Seconds, 0 if unknown. </summary>
public float Duration { get; set; }
public bool AudioOnly { get; set; }
public string Thumbnail { get; set; }
/// <summary> The stream is merged into local files by each client - it seeks by reopening at a time. </summary>
public bool Merged { get; set; }
/// <summary> A live stream (a Twitch channel...): played near its live edge, no seeking, no duration. </summary>
public bool IsLive { get; set; }
/// <summary> Video heights clients can pick, highest first. </summary>
public List<int> Qualities { get; set; } = new();
/// <summary> Audio tracks (dubs) clients can pick. Empty when there's no choice. </summary>
public List<MediaAudioTrack> AudioTracks { get; set; } = new();
/// <summary> Where to start playing (seconds), from the link (YouTube's <c>t=</c>). 0 = the beginning. </summary>
public float StartAt { get; set; }
/// <summary> Caption tracks clients can show. Empty when there are none. </summary>
public List<MediaCaptionTrack> CaptionTracks { get; set; } = new();
}
/// <summary>
/// What a client wants to play.
/// </summary>
public sealed class StreamRequest
{
public bool AudioOnly { get; set; }
/// <summary> The server's bimp_max_height. </summary>
public int MaxHeight { get; set; } = 720;
/// <summary> An exact height the client picked, 0 for automatic. </summary>
public int Height { get; set; }
/// <summary> Audio language the client picked, null for the original. </summary>
public string Language { get; set; }
/// <summary> A resolver client whose urls refused mid-download: re-resolving skips it, its fresh urls pass the probe and die the same way (see <see cref="StreamFile.ClientKey"/>). </summary>
public string AvoidClient { get; set; }
/// <summary> A copy of this request that avoids a client whose urls just refused. </summary>
public StreamRequest Avoiding( string clientKey ) => new() { AudioOnly = AudioOnly, MaxHeight = MaxHeight, Height = Height, Language = Language, AvoidClient = clientKey };
}
public enum StreamKind
{
/// <summary> A url the engine plays as-is (progressive mp4/webm/mp3). </summary>
Direct,
/// <summary> Separate video + audio WebM files, merged locally into segment files. </summary>
Merge,
/// <summary> A single audio WebM, re-laid out locally (cues at the end) for audio only playback. </summary>
Audio,
/// <summary> A finished HLS playlist (<see cref="StreamPlan.DirectUrl"/>), remuxed locally from a start time. </summary>
Hls,
/// <summary> A live stream url (<see cref="StreamPlan.DirectUrl"/>: HLS, MPEG-TS, RTSP), played like a pasted live link. </summary>
Live,
}
/// <summary>
/// One downloadable file of a stream.
/// </summary>
public sealed class StreamFile
{
public string Url { get; set; }
/// <summary> Bytes, 0 if unknown. </summary>
public long Size { get; set; }
/// <summary> Extractor specific id of the format (YouTube itag), to find the same file again after a refresh. </summary>
public string FormatId { get; set; }
/// <summary> "webm", or "mp4" for a fragmented MP4 (YouTube's AV1). </summary>
public string Container { get; set; } = "webm";
/// <summary> The resolver client that minted Url (null off YouTube). Media GETs identify as it, and a refusal re-resolves past it - a dead client's fresh urls pass the probe all the same. </summary>
public string ClientKey { get; set; }
}
/// <summary>
/// How a client plays a piece of media, resolved on that client.
/// </summary>
public sealed class StreamPlan
{
public StreamKind Kind { get; set; }
public StreamFile Video { get; set; }
public StreamFile Audio { get; set; }
/// <summary> A video to use instead if <see cref="Video"/> can't be read (the VP9 copy of an AV1 choice). </summary>
public StreamFile FallbackVideo { get; set; }
/// <summary>
/// A video to carry on with once <see cref="Video"/> runs out: YouTube serves its AV1 4K60 to the clients that get it
/// for only ~88 MB (about a minute), so playback switches to VP9 1440p60 at a keyframe (see WebmSegmenter).
/// </summary>
public StreamFile SwitchVideo { get; set; }
/// <summary> For <see cref="StreamKind.Direct"/>, <see cref="StreamKind.Hls"/> and <see cref="StreamKind.Live"/>. </summary>
public string DirectUrl { get; set; }
/// <summary> The client that minted the plan's urls - an AV1 <see cref="Video"/> can come from another one (see <see cref="StreamFile.ClientKey"/>). </summary>
public string ClientKey { get; set; }
}
/// <summary>
/// Understands one site. Stateless apart from caches, safe to share.
/// </summary>
public interface IExtractor
{
/// <summary> Short name, used in play tokens ("yt"). </summary>
string Key { get; }
bool CanHandle( Uri url );
/// <summary> Host: title, duration, choices. Throws <see cref="ResolveException"/>. </summary>
Task<MediaInfo> GetInfoAsync( Uri url, bool audioOnly, CancellationToken ct );
/// <summary> Client: the files to play. Throws <see cref="ResolveException"/>. </summary>
Task<StreamPlan> GetStreamsAsync( string id, StreamRequest request, CancellationToken ct );
}
/// <summary> An extractor whose media can have captions: <see cref="MediaInfo.CaptionTracks"/> lists them, this fetches one. </summary>
public interface ICaptionExtractor
{
/// <summary>
/// Client: the raw caption file of one track (<see cref="MediaCaptionTrack.Id"/>), for <see cref="Captions.Parse"/>.
/// Fetched per client because the urls can be tied to the requesting IP. Null when it can't be had.
/// </summary>
Task<string> GetCaptionsAsync( string id, string trackId, CancellationToken ct );
}
/// <summary> One video of a playlist. </summary>
public sealed record PlaylistEntry( string Url, string Title, float Duration );
/// <summary> An extractor that can list the videos of a playlist link. </summary>
public interface IPlaylistExtractor
{
/// <summary> The entries (at most <paramref name="max"/>) if this url is a playlist without a single video picked, else null. </summary>
Task<List<PlaylistEntry>> GetPlaylistAsync( Uri url, int max, CancellationToken ct );
}
Game
library
using Bimp.Resolver.Media;
namespace Bimp;
/// <summary>
/// Plays a series of local segment files back to back: merged YouTube media (<see cref="WebmSegmenter"/>) or a
/// live stream remuxed to MP4 (<see cref="Resolver.Live.LiveSegmenter"/>).
/// <para>
/// Segment N plays while N+1 is started just in time in a second, hidden and silent player; at the boundary
/// they swap, so the picture carries straight on. Segments keep absolute timestamps, so <see cref="Time"/>
/// is the media time.
/// </para>
/// </summary>
public sealed class SegmentPlayer : IDisposable
{
class Slot
{
public MediaSegment Segment;
public VideoPlayer Player;
public bool Loaded;
public bool Finished;
public bool Preload; // waiting in the wings: silent until it's swapped in
public bool FirstFrame; // set when it delivers its first frame (from OnTextureData, during Present)
public bool Primed; // its decoder has produced its first frame
public int Steady; // frames in a row it has delivered on time (see SettleFrames)
public RealTimeSince SinceFrame;
public bool Fading; // swapped out: silent (its volume ramps down) and closed a moment later
public float LastTime;
public RealTimeSince SinceOpened;
}
readonly ISegmentFeed feed;
/// <summary>
/// Live: start this many segments behind the newest finished one, so the next is always ready by the time the
/// current one ends. Latency is roughly (JoinBehind + 1) segments plus the overlap.
/// </summary>
int JoinBehind => feed.LowLatency ? 0 : 1;
/// <summary> Live: further behind than this (slow machine, stalled download) jumps back to the live edge. </summary>
int MaxBehind => feed.LowLatency ? 1 : 3;
/// <summary>
/// Low latency: join the newest segment this long after it's written, so the next one is ready a little before
/// the current one ends (its start latency plus arrival jitter). A segment that's late anyway just holds the
/// last picture (the filler frames) until it arrives.
/// </summary>
const double LowJoinMargin = 0.2;
/// <summary>
/// Average time from opening a segment to its first decoded frame (measured as we go).
/// </summary>
float startLatency;
/// <summary>
/// How long before the current segment ends to start the next one, so its first frame lands on the boundary -
/// and so the two players are close to in step for the audio crossfade. A player that isn't primed in time
/// just swaps a moment later (the current one plays on into its overlap, if the file has one).
/// </summary>
float StartLead => JustInTime
? Math.Clamp( (startLatency > 0 ? startLatency : 0.08f) + 0.015f, 0.0f, 1.0f )
: ClockLeadOnly ? Math.Clamp( clockLead, 0.0f, Math.Min( current.Segment.Overlap, 1.0f ) )
: Math.Clamp( (startLatency > 0 ? startLatency : 0.04f) + leadCorrection, 0.0f, feed.IsLive ? 1.0f : 3.0f );
/// <summary>
/// Media files that run on into the next segment (<see cref="MediaSegment.Overlap"/>): the current player shows real
/// frames until the next one's first frame is up, however long that takes (150-520 ms at 1440p/4K), so the next one
/// only has to be opened for its clock to be in step. Adding the start latency to that made it skip that far ahead
/// at a swap, and correcting that back made the current one hold its last frame instead.
/// <para>
/// Merged WebM (VP9/AV1) only: MP4 segments (HLS VOD) kept waiting for the next H.264 player to settle made Media
/// Foundation drop a sample ("MFT deadlock") in every run, and the picture broke up until the next keyframe (3 of 3
/// runs, against 0 of 3 with the old timing).
/// </para>
/// </summary>
bool ClockLeadOnly => feed is WebmSegmenter && (current?.Segment.Overlap ?? 0) > 0;
/// <summary> How long before the end to open the next player so its clock is in step (<see cref="ClockLeadOnly"/>), learned from each swap. </summary>
float clockLead = 0.05f;
/// <summary>
/// <see cref="ClockLeadOnly"/>: swap once the next player has delivered this many frames in a row on time. Its first
/// frame comes late, and a 1440p decoder is still catching up to its clock for a moment after it - swapping on the
/// first frame put that on screen as a 80-120 ms gap (measured). The current one plays on into its overlap meanwhile.
/// </summary>
const int SettleFrames = 4;
const float SettleInterval = 0.045f;
/// <summary>
/// Low latency files that end at their boundary (H.264, ending in skip frames): the next one just opens early enough
/// for its first frame to be ready there. (Syncing clocks made every swap ~40 ms late: a new player's clock runs from
/// Play(), its first frame comes ~60 ms later.)
/// </summary>
bool JustInTime => feed.LowLatency && (current?.Segment.Overlap ?? 0) <= 0;
/// <summary>
/// Learned from each swap: how far the next player's clock was ahead of the current one's at the boundary.
/// Starting it that much later next time brings them in step, which the audio crossfade needs - and so the next
/// player neither skips its first frames nor makes the current one hold its last, while the current one plays on
/// into its overlap (see <see cref="MediaSegment.Overlap"/>).
/// </summary>
float leadCorrection;
/// <summary>
/// How long the swapped-out player keeps running, silent, after a swap.
/// <para>
/// The engine's mixer runs every sound through Steam Audio's direct effect, whose gain interpolator ramps any
/// volume change over the next mix frames. Unmuting the next player while closing the current one outright
/// left a ~70 ms dip in the audio at every boundary (measured with a sine: down to 5%, every segment).
/// Muting the old player at the same moment instead - both ramps at once - makes that a crossfade.
/// </para>
/// </summary>
const float FadeSeconds = 0.15f;
Slot fading;
RealTimeSince sinceFade;
Slot current, next;
bool paused;
// audio settings, applied to both players
Vector3 audioPosition;
bool audioSpatial;
float audioVolume = 1;
float audioDistance = 2500;
string error;
public string Error => error ?? feed.Error;
/// <summary> The segment on screen, -1 before the first. </summary>
public int CurrentSeq => current?.Segment.Seq ?? -1;
/// <summary> Has the first segment started? </summary>
public bool Loaded => current is { Loaded: true } || (Mjpeg?.HasShown ?? false);
/// <summary> Motion JPEG: frames go straight to the texture, there are no segments to play. </summary>
Resolver.Live.LiveMjpeg Mjpeg => (feed as Resolver.Live.LiveSegmenter)?.Mjpeg;
/// <summary> The last segment has played to its end. </summary>
public bool Finished { get; private set; }
/// <summary> Media time (VOD), or running stream time (live). </summary>
public float Time => (float)StreamTime;
/// <summary>
/// Stream time (seconds) of what's on screen. WebM segments keep absolute timestamps (their StreamStart is 0); MP4
/// segments (live, HLS VOD) each start their clock at 0.
/// </summary>
public double StreamTime => Mjpeg is { } mjpeg ? mjpeg.ShownTime : (current?.Segment.StreamStart ?? 0) + (current?.Player?.PlaybackTime ?? 0);
public float Duration => current?.Player?.Duration ?? 0;
public bool IsPaused => current?.Player?.IsPaused ?? paused;
/// <summary> Every segment's frames go to one texture (see <see cref="VideoFrameSink"/>). </summary>
readonly VideoFrameSink sink = new();
public Texture Texture => sink.Texture;
public int Width => sink.Width;
public int Height => sink.Height;
public bool HasVideo => sink.Texture is not null && Width > 4 && Height > 4;
public (int rate, int channels) AudioFormat => (current?.Player?.SampleRate ?? 0, current?.Player?.Channels ?? 0);
/// <summary>
/// Merged media: the synced media time, null while paused. A first segment that starts later than it (at the next
/// keyframe - see <see cref="MediaBackend"/>) opens only when the clock gets there, so it never has to seek forward.
/// </summary>
public float? SyncedTime { get; set; }
public SegmentPlayer( ISegmentFeed feed )
{
this.feed = feed;
if ( !feed.IsLive ) feed.Want( 0 );
}
/// <summary>
/// Call every frame.
/// </summary>
public void Update()
{
if ( Mjpeg is { } mjpeg )
{
if ( !paused ) mjpeg.Present( sink );
return;
}
if ( current is null )
{
var first = feed.IsLive ? LiveJoinSegment() : feed.TryGet( 0 );
if ( first is null ) return;
if ( !feed.IsLive && !paused && SyncedTime is { } now && first.StreamStart + first.Start - now > StartLead ) return;
current = Open( first );
if ( paused ) current.Player?.Pause();
}
// Live and fell far behind? Jump to the live edge.
if ( feed.IsLive && feed.Newest - current.Segment.Seq > MaxBehind && LiveJoinSegment() is { } edge )
{
Log.Info( $"[bimp] live: {feed.Newest - current.Segment.Seq} segments behind, jumping to the live edge" );
Close( next ); next = null;
Close( current );
current = Open( edge );
}
TimedPresent( current, "current" );
// the first segment's own start seeds the start latency, so the first swap isn't late on a slow (4K) decoder
if ( startLatency <= 0 && !current.Primed && current.FirstFrame )
{
current.Primed = true;
startLatency = (float)current.SinceOpened;
}
// the swapped-out player finishes its fade, then goes
if ( fading is not null )
{
TimedPresent( fading, "fading" );
if ( sinceFade > FadeSeconds ) CloseFading();
}
var p = current.Player;
var seg = current.Segment;
// A player started fresh delivers its first frame ~0.25s after Play(), and one that was paused takes ~1s
// after Resume() - so start the next segment just in time (hidden, silent, presented so it runs),
// timed from the measured start latency, and swap which player is on screen. Nothing ever pauses.
if ( next is null && !paused && !seg.Last && current.Loaded && p is not null && p.PlaybackTime >= seg.End - StartLead )
{
var upcoming = feed.TryGet( seg.Seq + 1 );
if ( upcoming is not null ) next = Open( upcoming, preload: true );
}
if ( next?.Player is not null )
{
TimedPresent( next, "next" );
if ( !next.Primed && next.FirstFrame )
{
next.Primed = true;
var latency = (float)next.SinceOpened;
startLatency = startLatency <= 0 ? latency : startLatency * 0.7f + latency * 0.3f;
MediaProbe.Note( $"seg {next.Segment.Seq} first frame {latency * 1000:0}ms after opening (avg {startLatency * 1000:0}ms), current at {p?.PlaybackTime:0.000}/{seg.End:0.000}" );
}
}
// Swap when the current segment's clock reaches its end. (OnFinished fires ~80ms early, while
// frames are still queued - swapping on it cut off the last frames.)
var stuck = current.Finished && p is not null && MathF.Abs( p.PlaybackTime - current.LastTime ) < 0.0001f;
current.LastTime = p?.PlaybackTime ?? 0;
var atEnd = (current.Loaded && p is not null && p.PlaybackTime >= seg.End - 0.005f) || stuck;
if ( !atEnd ) return;
if ( seg.Last )
{
Finished = current.Finished || stuck;
return;
}
// a next player still catching up waits, while the current one has overlap left to play
var settled = !ClockLeadOnly || next is null || next.Steady >= SettleFrames || (p is not null && p.PlaybackTime >= seg.End + seg.Overlap - 0.1f) || stuck;
if ( next is { Primed: true } && settled )
{
// the next player's clock vs the current one's, in stream time: + means it's ahead. (Merged media keeps
// absolute timestamps; live segments each start their clock at 0.)
var ahead = (float)((next.Segment.StreamStart + (next.Player?.PlaybackTime ?? 0)) - (seg.StreamStart + (p?.PlaybackTime ?? seg.End)));
// keep the two clocks in step (see leadCorrection)
if ( ClockLeadOnly ) clockLead = Math.Clamp( clockLead - ahead * 0.7f, 0.0f, 1.0f );
else if ( !JustInTime ) leadCorrection = Math.Clamp( leadCorrection - ahead * 0.7f, -0.2f, 0.2f );
MediaProbe.Note( $"swap {seg.Seq}->{next.Segment.Seq} at {p?.PlaybackTime:0.000}/{seg.End:0.000} (overlap {seg.Overlap * 1000:0}ms), next ahead by {ahead * 1000:0}ms, lead now {StartLead * 1000:0}ms" );
next.Preload = false;
ApplyAudio( next );
// its frame from just before the boundary goes up now, not 33 ms later with its next one
sink.ShowHeld( next.Player );
// crossfade: the old player goes silent as the new one comes up (see FadeSeconds)
CloseFading();
fading = current;
fading.Fading = true;
ApplyAudio( fading );
sinceFade = 0;
current = next;
next = null;
feed.Want( current.Segment.Seq ); // keeps the one after it coming
}
// otherwise the next segment isn't downloaded yet - hold the last frame
}
void TimedPresent( Slot slot, string role )
{
if ( slot?.Player is null ) return;
var t = System.Diagnostics.Stopwatch.GetTimestamp();
slot.Player.Present();
MediaProbe.PresentTook( System.Diagnostics.Stopwatch.GetElapsedTime( t ).TotalMilliseconds, role, slot.Segment.Seq );
}
/// <summary>
/// Live: the segment to start from - the newest one that's at least <see cref="JoinBehind"/> behind the newest
/// finished segment and leaves the feed's start buffer of stream time after it. Null while there isn't enough yet.
/// </summary>
MediaSegment LiveJoinSegment()
{
var newest = feed.TryGet( feed.Newest );
if ( newest is null ) return null;
var bufferedUntil = newest.StreamStart + newest.End;
if ( feed.LowLatency ) return RealTime.NowDouble - newest.ReadyAt >= LowJoinMargin ? newest : null;
for ( var seq = newest.Seq - JoinBehind; seq >= 0; seq-- )
{
var s = feed.TryGet( seq );
if ( s is null ) return null; // not enough on disk yet
if ( bufferedUntil - s.StreamStart >= feed.StartBufferSeconds ) return s;
}
return null;
}
Slot Open( MediaSegment segment, bool preload = false )
{
MediaProbe.Note( $"open seg {segment.Seq} ({segment.Start:0.00}-{segment.End:0.00}){(preload ? " (preload)" : "")}" );
var slot = new Slot { Segment = segment, SinceOpened = 0, Preload = preload };
try
{
var player = new VideoPlayer();
player.OnLoaded += () => slot.Loaded = true;
player.OnFinished += () => slot.Finished = true;
slot.Player = player;
ApplyAudio( slot );
// only the player on screen updates the texture; a preloading one just says it's ready
sink.Attach( player, () => !slot.Preload && !slot.Fading, () =>
{
slot.Steady = slot.FirstFrame && slot.SinceFrame < SettleInterval ? slot.Steady + 1 : 0;
slot.SinceFrame = 0;
slot.FirstFrame = true;
}, $"seg {segment.Seq}", holdWhileInactive: preload );
var playTimer = System.Diagnostics.Stopwatch.StartNew();
player.Play( FileSystem.Data, segment.Path );
MediaProbe.Note( $"Play() seg {segment.Seq} took {playTimer.Elapsed.TotalMilliseconds:0.0}ms" );
}
catch ( Exception e )
{
error = e.Message;
}
return slot;
}
void Close( Slot slot )
{
if ( slot?.Player is null ) return;
var timer = System.Diagnostics.Stopwatch.StartNew();
sink.Detach( slot.Player );
slot.Player.Dispose();
MediaProbe.Note( $"close seg {slot.Segment.Seq} took {timer.Elapsed.TotalMilliseconds:0.0}ms" );
}
void CloseFading()
{
if ( fading is null ) return;
Close( fading );
feed.Release( fading.Segment.Seq );
fading = null;
}
public void SetPaused( bool pause )
{
paused = pause;
if ( current?.Player is { } p )
{
if ( pause ) p.Pause(); else p.Resume();
}
if ( pause ) CloseFading();
// a preloading player would carry on running - start it again when we resume
if ( pause && next is not null )
{
Close( next );
next = null;
}
}
/// <summary>
/// A short hop within the current segment (landing exactly on the synced time after opening).
/// Anything further is done by opening a new segmenter at the time.
/// </summary>
public void Seek( float time )
{
var seg = current?.Segment;
if ( seg is null ) return;
current.Player?.Seek( Math.Clamp( time - (float)seg.StreamStart, seg.Start, seg.End ) );
Finished = false;
}
public void SetAudio( Vector3 position, bool spatial, float volume, float distance )
{
audioPosition = position;
audioSpatial = spatial;
audioVolume = volume;
audioDistance = distance;
ApplyAudio( current );
ApplyAudio( next );
}
void ApplyAudio( Slot slot )
{
if ( slot?.Player is null ) return;
var a = slot.Player.Audio;
a.ListenLocal = !audioSpatial;
a.Position = audioSpatial ? audioPosition : Vector3.Forward * 64.0f;
a.Volume = slot.Preload || slot.Fading ? 0 : audioVolume;
a.Distance = audioDistance;
a.Falloff = MediaPlayer.AudioFalloff;
}
public void Dispose()
{
CloseFading();
Close( current );
Close( next );
current = next = null;
sink.Dispose();
}
}
Game
library
namespace Bimp;
/// <summary>
/// One USE key press can reach us several ways in the same frame: the screen's own look-and-press
/// (<see cref="MediaScreen"/>), the PlayerController pressing <see cref="MediaInteract"/> when in reach,
/// and the remote's close-on-use. Whoever runs first handles it, the rest see it's taken.
/// </summary>
internal static class UseHandled
{
static float handledAt = -1;
/// <summary>
/// Returns true if nobody has handled a USE press this frame (and marks it handled).
/// </summary>
public static bool TryConsume()
{
if ( handledAt == Time.Now ) return false;
handledAt = Time.Now;
return true;
}
}
Game
library
@using System
@using System.Linq
@using Sandbox
@using Sandbox.UI
@namespace Bimp
@inherits PanelComponent
@implements Component.ISceneStage
@*
The full controls, shown on the local player's screen when they USE a media player.
Created on demand by MediaRemote.Open, never networked.
*@
<root class="mediaremote @(PadMode ? "pad" : "")">
@if ( Target is not null )
{
<div class="window">
<div class="header">
<i>@(Target.AudioOnlyPlayer ? "speaker" : "smart_display")</i>
<div class="title">Media Player</div>
<div class="close nav" onclick="@Close"><i>close</i></div>
</div>
<div class="now">
@if ( Target.HasMedia )
{
<div class="label">Now playing</div>
<div class="track">@Target.Title</div>
<div class="sub">
@if ( !string.IsNullOrWhiteSpace( Target.RequestedBy ) )
{
<span>Requested by @Target.RequestedBy</span>
}
<span class="time">@TimeText</span>
</div>
}
else
{
<div class="track dim">Nothing playing</div>
}
@if ( !string.IsNullOrWhiteSpace( Target.Status ) )
{
<div class="status">@Target.Status</div>
}
@if ( !string.IsNullOrWhiteSpace( Target.LocalNotice ) )
{
<div class="notice">@Target.LocalNotice</div>
}
</div>
@if ( Target.HasMedia && !Target.IsLive && Target.MediaDuration > 0 )
{
<div class="seek @(Target.CanLocalControl ? "nav" : "disabled")">
<SliderControl Min="@(0f)" Max="@Target.MediaDuration" Step="@(1f)" ShowValueTooltip="@false" Value="@SeekValue" OnValueChanged="@OnSeekChanged"></SliderControl>
</div>
}
@if ( Target.CanLocalControl && Target.HasMedia )
{
<div class="transport">
@if ( !Target.IsLive )
{
<div class="btn nav" onclick="@( () => Target.RequestSeek( Target.CurrentTime - 10 ) )"><i>replay_10</i></div>
<div class="btn big nav nav-default" onclick="@( () => Target.RequestSetPaused( !Target.Paused ) )"><i>@(Target.Paused ? "play_arrow" : "pause")</i></div>
<div class="btn nav" onclick="@( () => Target.RequestSeek( Target.CurrentTime + 10 ) )"><i>forward_10</i></div>
}
<div class="btn nav" onclick="@Target.RequestSkip"><i>skip_next</i></div>
<div class="btn nav" onclick="@Target.RequestStop"><i>stop</i></div>
</div>
}
@if ( Target.HasMedia && (Target.AvailableQualities.Count > 0 || Target.AvailableAudioTracks.Count > 0 || Target.AvailableCaptionTracks.Count > 0) )
{
<div class="stream-options">
@if ( Target.AvailableQualities.Count > 0 )
{
<div class="option">
<i>high_quality</i>
<DropDown class="quality nav" Value="@QualityValue" ValueChanged="@OnQualityChanged" BuildOptions="@BuildQualityOptions"></DropDown>
</div>
}
@if ( Target.AvailableAudioTracks.Count > 0 )
{
<div class="option">
<i>translate</i>
<DropDown class="audio-track nav" Value="@AudioTrackValue" ValueChanged="@OnAudioTrackChanged" BuildOptions="@BuildAudioTrackOptions"></DropDown>
</div>
}
@if ( Target.AvailableCaptionTracks.Count > 0 )
{
<div class="option">
<i>closed_caption</i>
<DropDown class="audio-track nav" Value="@CaptionValue" ValueChanged="@OnCaptionChanged" BuildOptions="@BuildCaptionOptions"></DropDown>
</div>
}
<div class="playing-as">@PlayingAsText</div>
</div>
}
@if ( (Target.CanLocalControl || Target.CanLocalQueue) && PadMode )
{
@* a pad can't type a link *@
<div class="pad-note"><i>keyboard</i> Use a keyboard to add a link</div>
}
else if ( Target.CanLocalControl || Target.CanLocalQueue )
{
<div class="entry">
<TextEntry @ref="UrlEntry" Placeholder="Paste a YouTube link or a media url (mp4, webm, mp3...)" onsubmit="@OnSubmit"></TextEntry>
@if ( Target.CanLocalControl )
{
<div class="action primary" onclick="@PlayNow"><i>play_arrow</i> Play now</div>
}
@if ( Target.CanLocalQueue )
{
<div class="action" onclick="@AddToQueue"><i>playlist_add</i> Queue</div>
}
</div>
}
else
{
<div class="locked"><i>lock</i> You can't control this player.</div>
}
<div class="queue">
<div class="queue-header">
<div class="tab nav @(ShowRecent ? "" : "active")" onclick="@( () => ShowRecent = false )">Up next <span class="count">@Target.Queue.Count / @Target.MaxQueue</span></div>
<div class="tab nav @(ShowRecent ? "active" : "")" onclick="@( () => ShowRecent = true )">Recent <span class="count">@RecentLinks.All.Count</span></div>
<div class="spacer"></div>
@if ( !ShowRecent && Target.CanLocalControl && Target.Queue.Count > 0 )
{
<div class="small-btn nav" onclick="@Target.RequestClearQueue">Clear</div>
}
</div>
<div class="queue-list">
@if ( ShowRecent )
{
@if ( RecentLinks.All.Count == 0 )
{
<div class="empty">Links you play show up here.</div>
}
@foreach ( var recent in RecentLinks.All )
{
var url = recent.Url;
<div class="queue-item recent">
<div class="info">
<div class="name">@recent.Title</div>
<div class="by">@recent.Url</div>
</div>
<div class="dur">@(recent.IsLive ? "Live" : recent.Duration > 0 ? MediaSource.FormatTime( recent.Duration ) : "")</div>
@if ( Target.CanLocalControl )
{
<div class="small-btn go nav" onclick="@( () => Target.RequestPlay( url ) )"><i>play_arrow</i></div>
}
@if ( Target.CanLocalQueue )
{
<div class="small-btn go nav" onclick="@( () => Target.RequestEnqueue( url ) )"><i>playlist_add</i></div>
}
<div class="small-btn nav" onclick="@( () => RecentLinks.Remove( url ) )"><i>close</i></div>
</div>
}
}
else if ( Target.Queue.Count == 0 )
{
<div class="empty">The queue is empty.</div>
}
@for ( int i = 0; !ShowRecent && i < Target.Queue.Count; i++ )
{
var index = i;
var item = Target.Queue[i];
<div class="queue-item">
<div class="num">@(index + 1)</div>
<div class="info">
<div class="name">@item.Title</div>
<div class="by">@item.RequestedBy</div>
</div>
<div class="dur">@(item.IsLive ? "Live" : item.Duration > 0 ? MediaSource.FormatTime( item.Duration ) : "")</div>
@if ( Target.CanLocalControl )
{
<div class="small-btn nav" onclick="@( () => Target.RequestRemove( index ) )"><i>close</i></div>
}
</div>
}
</div>
</div>
<div class="footer">
<div class="btn small nav" onclick="@ToggleMute"><i>@(MediaSettings.Mute ? "volume_off" : "volume_up")</i></div>
<SliderControl class="volume nav" Min="@(0f)" Max="@(1f)" Step="@(0.01f)" ShowValueTooltip="@false" Value="@MediaSettings.Volume" OnValueChanged="@OnVolumeChanged"></SliderControl>
<div class="btn small nav audio-mode @(MediaSettings.SpatialAudio ? "" : "flat")" onclick="@ToggleSpatial" tooltip="@(MediaSettings.SpatialAudio ? "3D audio: sound comes from the screen. Click for 2D" : "2D audio: no direction, still quieter further away. Click for 3D")"><i>@(MediaSettings.SpatialAudio ? "surround_sound" : "headphones")</i><span>@(MediaSettings.SpatialAudio ? "3D" : "2D")</span></div>
@if ( PadMode )
{
<div class="hint">
<InputGlyph Analog="LeftStickY" Size="@InputGlyphSize.Small"></InputGlyph><span>Move</span>
@if ( Pad.SelectAction is not null )
{
<InputGlyph Action="@Pad.SelectAction" Size="@InputGlyphSize.Small"></InputGlyph><span>Select</span>
}
<InputGlyph Action="@(Pad.BackAction ?? "use")" Size="@InputGlyphSize.Small"></InputGlyph><span>Close</span>
</div>
}
else
{
<div class="hint"><span>Your audio (all players) ·</span><InputGlyph Action="use" Size="@InputGlyphSize.Small"></InputGlyph><span>or</span><InputGlyph Key="escape" Size="@InputGlyphSize.Small"></InputGlyph><span>to close</span></div>
}
</div>
</div>
}
</root>
@code
{
/// <summary>
/// The player being controlled.
/// </summary>
public MediaPlayer Target { get; set; }
/// <summary>
/// Close automatically when the camera gets further than this from the player.
/// </summary>
public float MaxDistance { get; set; } = 1500.0f;
TextEntry UrlEntry { get; set; }
/// <summary> The list shows the links this client has played (<see cref="RecentLinks"/>) instead of the queue. </summary>
bool ShowRecent { get; set; }
RealTimeSince sinceOpened = 0;
float? pendingSeek;
RealTimeSince sincePendingSeek;
float SeekValue => pendingSeek ?? Target?.CurrentTime ?? 0;
string TimeText
{
get
{
if ( Target is null || !Target.HasMedia ) return "";
if ( Target.IsLive ) return "Live";
var cur = MediaSource.FormatTime( Target.CurrentTime );
var s = Target.MediaDuration > 0 ? $"{cur} / {MediaSource.FormatTime( Target.MediaDuration )}" : cur;
return Target.Paused ? $"{s} (paused)" : s;
}
}
/// <summary>
/// Open the remote for this player, or close it if it's already open for it.
/// </summary>
public static void Toggle( MediaPlayer target )
{
if ( !target.IsValid() || Application.IsDedicatedServer ) return;
var existing = target.Scene.GetAllComponents<MediaRemote>().FirstOrDefault();
if ( existing.IsValid() )
{
if ( existing.Target == target )
{
existing.Close();
return;
}
existing.Target = target;
existing.sinceOpened = 0;
existing.StateHasChanged();
return;
}
var go = target.Scene.CreateObject();
go.Name = "bimp remote";
go.NetworkMode = NetworkMode.Never;
go.Flags |= GameObjectFlags.NotSaved | GameObjectFlags.Hidden;
var screen = go.Components.Create<ScreenPanel>();
screen.ZIndex = 200;
var remote = go.Components.Create<MediaRemote>();
remote.Target = target;
}
/// <summary>
/// Is a remote open right now? Games can use this to e.g. hide their crosshair.
/// </summary>
public static bool IsOpen( Scene scene ) => scene?.GetAllComponents<MediaRemote>().Any() ?? false;
public void Close()
{
GameObject.Destroy();
}
protected override void OnTreeFirstBuilt()
{
base.OnTreeFirstBuilt();
UrlEntry?.Focus();
}
//
// Controller: the D-pad / left stick moves a highlight between the controls (the "nav" ones), A presses,
// left/right adjusts sliders and choices, B closes. See PadNavigation.
//
/// <summary> Played with a controller right now (the engine switches as soon as they touch the keyboard or pad). </summary>
bool PadMode => Input.UsingController;
readonly PadNavigation Pad = new();
/// <summary> The engine's virtual cursor (left stick moves a pointer) is off while we navigate; this was its setting. </summary>
bool? virtualCursorWas;
/// <summary>
/// The very first thing in the frame, before anything (fixed update included) reads input: navigate, then suppress the
/// frame's input, so the pad doesn't also move, jump or duck the player behind the remote.
/// </summary>
void Component.ISceneStage.Start()
{
if ( !PadMode || !Target.IsValid() || Panel is null )
{
RestoreVirtualCursor();
return;
}
virtualCursorWas ??= Input.EnableVirtualCursor;
Input.EnableVirtualCursor = false;
Pad.Press ??= PressControl;
Pad.Adjust ??= AdjustControl;
Pad.Update( Panel );
// B closes (unless it just closed a drop down's menu), and USE again, as on keyboard
var close = sinceOpened > 0.2f && (Pad.BackPressed || Input.Pressed( "use" ));
Input.Suppressed = true;
if ( close ) Close();
}
/// <summary> The window's width in the stylesheet (.window), at 1080p. </summary>
const float WindowWidth = 760;
/// <summary>
/// The screen panel already scales the remote with the screen height (as if every screen were 1080 high); this sizes
/// it up from there by bimp_ui_scale - at its design size it was small at every resolution - but never wider than
/// 90% of the screen.
/// </summary>
void FitToScreen()
{
if ( Components.Get<ScreenPanel>() is not { } screen ) return;
var heightScale = Screen.Size.y / 1080.0f;
var widest = heightScale > 0 ? Screen.Size.x * 0.9f / (WindowWidth * heightScale) : 1;
var scale = Math.Clamp( MediaSettings.RemoteScale, 0.5f, Math.Max( 0.5f, widest ) );
if ( MathF.Abs( screen.Scale - scale ) > 0.001f ) screen.Scale = scale;
}
/// <summary> For probes: each option of an open menu, its classes, and * on the pad's selection. </summary>
IEnumerable<string> MenuOptions()
{
var top = Panel;
while ( top?.Parent is not null ) top = top.Parent;
var menu = top?.Descendants.OfType<Popup>().FirstOrDefault();
return menu is null ? Enumerable.Empty<string>() : menu.Children.Select( c => $"{(c as Button)?.Text}:{c.Classes}{(c == menu.SelectedChild ? "*" : "")}" );
}
/// <summary> For probes: the controller state and what's highlighted. </summary>
internal string DescribePad() => $"usingController={Input.UsingController} controllers={Input.ControllerCount} select={Pad.SelectAction ?? "-"} back={Pad.BackAction ?? "-"} virtualCursor={Input.EnableVirtualCursor} screen={Screen.Size} desktopScale={Screen.DesktopScale} uiScale={Panel?.ScaleToScreen} window={Panel?.Descendants.FirstOrDefault( p => p.HasClass( "window" ) )?.Box.Rect} focused=[{Pad.Focused?.Classes}] menu=[{string.Join( " | ", MenuOptions() )}]" + '\n' +
string.Join( '\n', Panel.Descendants.Where( p => p.HasClass( "nav" ) ).Select( p => $" [{p.Classes}] visible={p.IsVisible} rect={p.Box.Rect}" ) );
/// <summary> For probes: move the highlight as the pad would (up, down, left, right, or "a" to press it), and say where it went. </summary>
internal string ProbeMove( string direction )
{
if ( direction == "a" )
{
if ( Pad.Focused is { } focused ) PressControl( focused );
return $"a on [{Pad.Focused?.Classes}]";
}
var d = direction switch { "up" => new Vector2( 0, -1 ), "down" => new Vector2( 0, 1 ), "left" => new Vector2( -1, 0 ), _ => new Vector2( 1, 0 ) };
Pad.ProbeStep( Panel, d );
return $"{direction} -> [{Pad.Focused?.Classes}] volume={MediaSettings.Volume:0.00}";
}
void RestoreVirtualCursor()
{
if ( virtualCursorWas is not { } was ) return;
Input.EnableVirtualCursor = was;
virtualCursorWas = null;
}
protected override void OnDestroy()
{
RestoreVirtualCursor();
base.OnDestroy();
}
/// <summary> A on a control: a drop down opens its menu (navigated by PadNavigation), a button is clicked. Sliders only take left/right. </summary>
void PressControl( Panel control )
{
if ( control.HasClass( "seek" ) || control.HasClass( "volume" ) ) return;
if ( control is DropDown dropDown )
{
dropDown.Open();
return;
}
control.CreateEvent( new MousePanelEvent( "onclick", control, "mouseleft" ) );
}
/// <summary> Left/right on a slider: seek by 10 s, volume by 5%. Anything else moves left/right. </summary>
bool AdjustControl( Panel control, int step )
{
if ( control.HasClass( "seek" ) )
{
OnSeekChanged( Math.Clamp( SeekValue + step * 10, 0, Target.MediaDuration ) );
return true;
}
if ( control.HasClass( "volume" ) )
{
OnVolumeChanged( MathF.Round( (MediaSettings.Volume + step * 0.05f) * 20 ) / 20 );
return true;
}
return false;
}
protected override void OnUpdate()
{
if ( !Target.IsValid() )
{
Close();
return;
}
MenuStyle.Apply( Panel );
FitToScreen();
if ( Input.EscapePressed )
{
Input.EscapePressed = false;
Close();
return;
}
// USE again closes it (not while typing - a focused text entry takes the keys)
if ( sinceOpened > 0.2f && Input.Pressed( "use" ) && UseHandled.TryConsume() )
{
Close();
return;
}
if ( Scene.Camera is not null && Scene.Camera.WorldPosition.Distance( Target.WorldPosition ) > MaxDistance )
{
Close();
return;
}
// Seek once the slider has settled, rather than spamming the host while dragging
if ( pendingSeek.HasValue && sincePendingSeek > 0.3f )
{
Target.RequestSeek( pendingSeek.Value );
pendingSeek = null;
}
}
void OnSeekChanged( float value )
{
if ( Target is null || !Target.CanLocalControl ) return;
if ( MathF.Abs( value - Target.CurrentTime ) < 1.5f && !pendingSeek.HasValue ) return; // our own refresh
pendingSeek = value;
sincePendingSeek = 0;
}
//
// Resolution / audio track. These are this player's own preferences (saved convars), applied to every
// media player - picking one reloads the local stream at the same synced time.
//
string QualityValue => MediaSettings.Quality <= 0 ? "0" : Target.SelectedQuality.ToString();
List<Option> BuildQualityOptions()
{
var auto = $"Auto (up to {MediaSettings.MaxVideoHeight}p)";
var list = new List<Option> { new Option( auto, "0" ) };
foreach ( var h in Target.AvailableQualities )
list.Add( new Option( MediaStreamOptions.QualityLabel( h ), h.ToString() ) );
return list;
}
void OnQualityChanged( string value )
{
if ( !int.TryParse( value, out var h ) ) return;
if ( h == (MediaSettings.Quality <= 0 ? 0 : Target.SelectedQuality) ) return;
MediaSettings.Quality = h;
}
string AudioTrackValue
{
get
{
var selected = Target.SelectedAudioTrack ?? Target.AvailableAudioTracks.FirstOrDefault( t => t.IsDefault );
return selected?.Id ?? "";
}
}
List<Option> BuildAudioTrackOptions()
{
return Target.AvailableAudioTracks
.Select( t => new Option( t.IsDefault && !t.Name.Contains( "original", StringComparison.OrdinalIgnoreCase ) ? $"{t.Name} (original)" : t.Name, t.Id ) )
.ToList();
}
void OnAudioTrackChanged( string id )
{
if ( id == AudioTrackValue ) return;
var track = Target.AvailableAudioTracks.FirstOrDefault( t => t.Id == id );
// picking the original clears the preference, so dubbed videos default to original again
MediaSettings.AudioLanguage = track is null || track.IsDefault ? "" : track.Id;
}
//
// Captions: off, or a language. The choice is this player's own (saved convars) and applies to every media player.
// Auto-generated tracks are only offered for languages without a written one - that's what picking a language gets.
//
IEnumerable<MediaCaptionTrack> ListedCaptionTracks
=> Target.AvailableCaptionTracks.Where( t => !t.IsAuto || !Target.AvailableCaptionTracks.Any( w => !w.IsAuto && w.Lang == t.Lang ) );
string CaptionValue => Target.SelectedCaptionTrack?.Id ?? "";
List<Option> BuildCaptionOptions()
{
var list = new List<Option> { new Option( "Captions off", "" ) };
foreach ( var t in ListedCaptionTracks )
list.Add( new Option( t.IsAuto && !t.Name.Contains( "auto", StringComparison.OrdinalIgnoreCase ) ? $"{t.Name} (auto-generated)" : t.Name, t.Id ) );
return list;
}
void OnCaptionChanged( string id )
{
if ( id == CaptionValue ) return;
var track = Target.AvailableCaptionTracks.FirstOrDefault( t => t.Id == id );
MediaSettings.Subtitles = track is not null;
if ( track is not null ) MediaSettings.SubtitleLanguage = track.Lang;
}
string PlayingAsText
{
get
{
var b = Target.Backend;
if ( b is null || !b.Loaded ) return "Loading...";
return b.HasVideo ? $"Playing {b.Height}p" : "";
}
}
void OnVolumeChanged( float value )
{
MediaSettings.Volume = value.Clamp( 0, 1 );
if ( value > 0 ) MediaSettings.Mute = false;
}
void ToggleMute()
{
MediaSettings.Mute = !MediaSettings.Mute;
}
/// <summary> 3D audio from every media player, or flat 2D for this player (bimp_spatial). </summary>
void ToggleSpatial()
{
MediaSettings.SpatialAudio = !MediaSettings.SpatialAudio;
}
string TakeUrl()
{
var text = UrlEntry?.Text?.Trim();
if ( UrlEntry is not null ) UrlEntry.Text = "";
return string.IsNullOrWhiteSpace( text ) ? null : text;
}
void OnSubmit()
{
// Enter plays right away if we can, otherwise queues
if ( Target.CanLocalControl && !Target.HasMedia ) PlayNow();
else if ( Target.CanLocalQueue ) AddToQueue();
else PlayNow();
}
void PlayNow()
{
var url = TakeUrl();
if ( url is null ) return;
Target.RequestPlay( url );
}
void AddToQueue()
{
var url = TakeUrl();
if ( url is null ) return;
Target.RequestEnqueue( url );
}
protected override int BuildHash()
{
if ( Target is null ) return 0;
return HashCode.Combine(
HashCode.Combine( Target.PlayId, Target.Title, Target.Paused, Target.Status, Target.LocalNotice, (int)Target.CurrentTime, Target.MediaDuration ),
HashCode.Combine( QueueHash(), MediaSettings.Mute, MediaSettings.Volume, MediaSettings.SpatialAudio, Target.CanLocalControl, pendingSeek, PadMode ),
HashCode.Combine( Target.Qualities, Target.AudioTracks, MediaSettings.Quality, MediaSettings.AudioLanguage, PlayingAsText, ShowRecent, RecentLinks.Version ),
HashCode.Combine( Target.CaptionTracks, MediaSettings.Subtitles, MediaSettings.SubtitleLanguage )
);
}
int QueueHash()
{
var h = new HashCode();
h.Add( Target.Queue.Count );
foreach ( var item in Target.Queue ) h.Add( item?.Url );
return h.ToHashCode();
}
}
Game
library
@using System
@using Sandbox
@using Sandbox.UI
@namespace Bimp
@inherits PanelComponent
@implements Component.ExecuteInEditor
@*
A small display for audio-only players (speakers, radios, jukeboxes).
Put this on a GameObject with a WorldPanel (PanelSize ~800x400).
*@
<root class="mediaspeaker @(Player?.HasMedia == true ? "playing" : "idle")">
<div class="header">
<i>@(Player?.HasMedia == true ? "graphic_eq" : "speaker")</i>
@if ( ShowUseHint )
{
<div class="title hint"><span>Press</span><InputGlyph Action="use"></InputGlyph><span>to play something</span></div>
}
else
{
<ScrollingText class="title" Text="@TitleText"></ScrollingText>
}
</div>
<SpectrumBars Player="@Player"></SpectrumBars>
<div class="footer">
<div class="time">@TimeText</div>
@if ( Player is not null && Player.Queue.Count > 0 )
{
<div class="queue"><i>queue_music</i> @Player.Queue.Count</div>
}
</div>
@if ( !string.IsNullOrWhiteSpace( Player?.LocalNotice ) )
{
<div class="notice">@Player.LocalNotice</div>
}
</root>
@code
{
/// <summary>
/// The player this display shows. Found automatically in this object or its parents if unset.
/// </summary>
[Property] public MediaPlayer Player { get; set; }
/// <summary>
/// Scale the UI to fit the WorldPanel's PanelSize instead of getting cut off. Sets the WorldPanel's RenderScale.
/// </summary>
[Property, Group( "Layout" )] public bool FitToPanel { get; set; } = true;
/// <summary>
/// The panel size the UI is designed for.
/// </summary>
[Property, Group( "Layout" ), ShowIf( nameof( FitToPanel ), true )] public Vector2 DesignSize { get; set; } = new Vector2( 560, 280 );
string TitleText
{
get
{
if ( Player is null ) return "";
if ( Player.HasMedia ) return Player.Title;
if ( !string.IsNullOrWhiteSpace( Player.Status ) ) return Player.Status;
return "";
}
}
/// <summary> Nothing playing or happening: show how to start something, with the USE button's glyph. </summary>
bool ShowUseHint => Player is not null && !Player.HasMedia && string.IsNullOrWhiteSpace( Player.Status );
string TimeText
{
get
{
if ( Player is null || !Player.HasMedia ) return "";
if ( Player.IsLive ) return "Live";
var cur = MediaSource.FormatTime( Player.CurrentTime );
var s = Player.MediaDuration > 0 ? $"{cur} / {MediaSource.FormatTime( Player.MediaDuration )}" : cur;
return Player.Paused ? $"{s} (paused)" : s;
}
}
protected override void OnUpdate()
{
if ( FitToPanel ) PanelFit.Apply( this, DesignSize );
if ( Scene.IsEditor ) return;
Player ??= GetComponentInParent<MediaPlayer>();
}
protected override int BuildHash()
{
if ( Player is null ) return 0;
return HashCode.Combine( Player.PlayId, Player.Title, Player.Status, Player.Paused, (int)Player.CurrentTime, Player.Queue.Count, Player.LocalNotice );
}
}
Game
library
namespace Bimp;
/// <summary>
/// Lets players use the media player with the USE key (works with the engine's <see cref="PlayerController"/>,
/// or anything else that calls <see cref="Component.IPressable"/>). Needs a collider on this GameObject or a child.
/// <para>
/// Aiming at one of the screen's on-screen buttons while the controls are showing presses that button.
/// Anywhere else (or while the controls are hidden) opens the full remote.
/// </para>
/// </summary>
[Title( "Media Player Interact" ), Category( "Media" ), Icon( "touch_app" )]
public sealed class MediaInteract : Component, Component.IPressable
{
/// <summary>
/// The player to control. Found automatically in this object or its parents if unset.
/// </summary>
[Property] public MediaPlayer Player { get; set; }
/// <summary>
/// The screen whose on-screen buttons USE can press. Found automatically in this object or its children if unset.
/// </summary>
[Property] public MediaScreen Screen { get; set; }
MediaPlayer Target => Player.IsValid() ? Player : (Player = GetComponentInParent<MediaPlayer>());
MediaScreen TargetScreen => Screen.IsValid() ? Screen : (Screen = GetComponentInChildren<MediaScreen>());
public bool CanPress( IPressable.Event e ) => Target.IsValid();
public bool Press( IPressable.Event e )
{
if ( !Target.IsValid() ) return false;
// The screen may already have handled this USE press itself this frame
if ( !UseHandled.TryConsume() ) return true;
// Press runs on the client that pressed, so this only affects them (button presses go through the
// player's permission-checked host requests)
var screen = TargetScreen;
if ( screen.IsValid() && screen.TryPressAimedControl() )
return true;
MediaRemote.Toggle( Target );
return true;
}
public IPressable.Tooltip? GetTooltip( IPressable.Event e )
{
if ( !Target.IsValid() ) return null;
var title = Target.HasMedia ? Target.Title : "Media Player";
var icon = Target.AudioOnlyPlayer ? "speaker" : "smart_display";
var action = TargetScreen.IsValid() ? TargetScreen.AimedActionName : null;
if ( action is not null )
return new IPressable.Tooltip( action, "touch_app", title );
return new IPressable.Tooltip( title, icon, "Open controls" );
}
}
Game
library
@using System
@using Sandbox
@using Sandbox.UI
@namespace Bimp
@inherits PanelComponent
@implements Component.ExecuteInEditor
@*
A small display for audio-only players (speakers, radios, jukeboxes).
Put this on a GameObject with a WorldPanel (PanelSize ~800x400).
*@
<root class="mediaspeaker @(Player?.HasMedia == true ? "playing" : "idle")">
<div class="header">
<i>@(Player?.HasMedia == true ? "graphic_eq" : "speaker")</i>
@if ( ShowUseHint )
{
<div class="title hint"><span>Press</span><InputGlyph Action="use"></InputGlyph><span>to play something</span></div>
}
else
{
<ScrollingText class="title" Text="@TitleText"></ScrollingText>
}
</div>
<SpectrumBars Player="@Player"></SpectrumBars>
<div class="footer">
<div class="time">@TimeText</div>
@if ( Player is not null && Player.Queue.Count > 0 )
{
<div class="queue"><i>queue_music</i> @Player.Queue.Count</div>
}
</div>
@if ( !string.IsNullOrWhiteSpace( Player?.LocalNotice ) )
{
<div class="notice">@Player.LocalNotice</div>
}
</root>
@code
{
/// <summary>
/// The player this display shows. Found automatically in this object or its parents if unset.
/// </summary>
[Property] public MediaPlayer Player { get; set; }
/// <summary>
/// Scale the UI to fit the WorldPanel's PanelSize instead of getting cut off. Sets the WorldPanel's RenderScale.
/// </summary>
[Property, Group( "Layout" )] public bool FitToPanel { get; set; } = true;
/// <summary>
/// The panel size the UI is designed for.
/// </summary>
[Property, Group( "Layout" ), ShowIf( nameof( FitToPanel ), true )] public Vector2 DesignSize { get; set; } = new Vector2( 560, 280 );
string TitleText
{
get
{
if ( Player is null ) return "";
if ( Player.HasMedia ) return Player.Title;
if ( !string.IsNullOrWhiteSpace( Player.Status ) ) return Player.Status;
return "";
}
}
/// <summary> Nothing playing or happening: show how to start something, with the USE button's glyph. </summary>
bool ShowUseHint => Player is not null && !Player.HasMedia && string.IsNullOrWhiteSpace( Player.Status );
string TimeText
{
get
{
if ( Player is null || !Player.HasMedia ) return "";
if ( Player.IsLive ) return "Live";
var cur = MediaSource.FormatTime( Player.CurrentTime );
var s = Player.MediaDuration > 0 ? $"{cur} / {MediaSource.FormatTime( Player.MediaDuration )}" : cur;
return Player.Paused ? $"{s} (paused)" : s;
}
}
protected override void OnUpdate()
{
if ( FitToPanel ) PanelFit.Apply( this, DesignSize );
if ( Scene.IsEditor ) return;
Player ??= GetComponentInParent<MediaPlayer>();
}
protected override int BuildHash()
{
if ( Player is null ) return 0;
return HashCode.Combine( Player.PlayId, Player.Title, Player.Status, Player.Paused, (int)Player.CurrentTime, Player.Queue.Count, Player.LocalNotice );
}
}
Game
library
.mediaspeaker
{
width: 100%;
height: 100%;
flex-direction: column;
padding: 28px 36px;
border-radius: 24px;
background: linear-gradient( to bottom, #151a24, #07090d );
font-family: Poppins;
color: white;
position: relative;
i
{
font-family: "Material Icons";
}
.header
{
flex-direction: row;
align-items: center;
gap: 16px;
i
{
font-size: 56px;
color: #3fa9ff;
}
.title
{
flex-grow: 1;
flex-basis: 0;
font-size: 40px;
font-weight: 700;
}
.hint
{
flex-direction: row;
align-items: center;
gap: 10px;
font-size: 28px;
font-weight: 600;
opacity: 0.8;
white-space: nowrap;
span
{
flex-shrink: 0;
}
}
}
.spectrumbars
{
flex-grow: 1;
margin: 20px 0;
}
.footer
{
flex-direction: row;
font-size: 28px;
opacity: 0.75;
.time
{
flex-grow: 1;
// every digit the same width, so it doesn't jiggle as it counts (Poppins has no tabular digits, Inter does)
font-family: Inter;
font-variant-numeric: tabular-nums;
}
.queue
{
align-items: center;
gap: 6px;
}
}
.notice
{
position: absolute;
left: 20px;
right: 20px;
bottom: 20px;
padding: 12px 20px;
border-radius: 12px;
background-color: rgba( 180, 40, 40, 0.92 );
font-size: 26px;
}
&.idle .spectrumbars
{
opacity: 0.3;
}
}
.spectrumbars
{
flex-direction: row;
align-items: flex-end;
gap: 6px;
.bar
{
flex-grow: 1;
height: 3%;
border-radius: 4px 4px 0 0;
background: linear-gradient( to top, #3fa9ff, #a36bff );
}
}
Game
library
using System;
namespace Bimp.Resolver.Live;
/// <summary>
/// AAC-LC decoder (ISO/IEC 14496-3 GA, 1024-sample frames): one raw access unit in, interleaved 16-bit PCM out.
/// The engine only decodes AAC inside finished MP4 files, which ties live audio to the segment latency - this
/// lets low-latency live streams play their AAC beside the video, like G.711 (see <see cref="LivePcmAudio"/>).
/// <para>
/// Supports SCE/CPE/LFE, section and scalefactor data, pulse data, M/S and intensity stereo, PNS, TNS, and the
/// long / start / eight-short / stop windows with sine and KBD shapes. HE-AAC decodes its AAC-LC core (half
/// band, SBR ignored). Main/LTP prediction and coupling channels aren't supported. Output is mono or stereo
/// (anything with more channels is folded down to its front three).
/// </para>
/// </summary>
public sealed class AacDecoder
{
const int ZeroHcb = 0, EscHcb = 11, NoiseHcb = 13, IntensityHcb2 = 14, IntensityHcb = 15;
const int OnlyLong = 0, LongStart = 1, EightShort = 2, LongStop = 3;
public int SampleRate { get; }
/// <summary> Output channels: 1 or 2. </summary>
public int Channels { get; }
readonly int rateIndex;
readonly short[] swbLong, swbShort;
readonly int numSwbLong, numSwbShort;
// per channel state: the second half of the last frame, and its window shape
readonly float[][] overlap = { new float[1024], new float[1024], new float[1024], new float[1024] };
readonly int[] previousShape = new int[4];
// scratch
readonly float[][] spectrum = { new float[1024], new float[1024] };
readonly float[] timeBuffer = new float[2048];
readonly float[][] pcm = { new float[1024], new float[1024], new float[1024], new float[1024] };
uint noiseState = 0x1f2e3d4c;
readonly Ics icsA = new(), icsB = new();
public AacDecoder( AacConfig config )
{
var asc = config.Asc;
var r = new BitReader( asc );
var objectType = ReadObjectType( r );
var index = r.Read( 4 );
var rate = index == 15 ? r.Read( 24 ) : AacConfig.RateOf( index );
var channelConfig = r.Read( 4 );
// HE-AAC signalled explicitly: the core is described after the SBR rate
if ( objectType is 5 or 29 )
{
var extIndex = r.Read( 4 );
if ( extIndex == 15 ) r.Read( 24 );
objectType = ReadObjectType( r );
}
if ( objectType != 2 ) throw new NotSupportedException( $"AAC object type {objectType} (only AAC-LC)" );
if ( r.Read( 1 ) == 1 ) throw new NotSupportedException( "960-sample AAC frames" );
rateIndex = index == 15 ? NearestRateIndex( rate ) : index;
SampleRate = rate > 0 ? rate : AacConfig.RateOf( rateIndex );
Channels = channelConfig == 1 ? 1 : 2;
(swbLong, swbShort) = rateIndex switch
{
0 or 1 => (AacTables.Sfb96_1024, AacTables.Sfb96_128),
2 => (AacTables.Sfb64_1024, AacTables.Sfb64_128),
3 or 4 => (AacTables.Sfb48_1024, AacTables.Sfb48_128),
5 => (AacTables.Sfb32_1024, AacTables.Sfb48_128),
6 or 7 => (AacTables.Sfb24_1024, AacTables.Sfb24_128),
8 or 9 or 10 => (AacTables.Sfb16_1024, AacTables.Sfb16_128),
_ => (AacTables.Sfb8_1024, AacTables.Sfb8_128),
};
numSwbLong = swbLong.Length - 1;
numSwbShort = swbShort.Length - 1;
}
static int ReadObjectType( BitReader r )
{
var t = r.Read( 5 );
return t == 31 ? 32 + r.Read( 6 ) : t;
}
static int NearestRateIndex( int rate )
{
int best = 3, bestDiff = int.MaxValue;
for ( int i = 0; i < 12; i++ )
{
var d = Math.Abs( AacConfig.RateOf( i ) - rate );
if ( d < bestDiff ) { best = i; bestDiff = d; }
}
return best;
}
/// <summary>
/// Decode one access unit (a raw_data_block) to 1024 interleaved samples per channel. Null if the frame can't be
/// decoded (it's skipped, the next one starts clean enough).
/// </summary>
public short[] Decode( byte[] frame )
{
var r = new BitReader( frame );
var outputs = 0; // decoded channels, in pcm[]
int centre = -1, left = -1, right = -1;
try
{
while ( r.BitsLeft >= 3 )
{
var id = r.Read( 3 );
if ( id == 7 ) break; // END
switch ( id )
{
case 0: // SCE
case 3: // LFE
r.Read( 4 );
if ( outputs < 4 )
{
var ics = icsA;
DecodeIcs( r, ics, false );
Reconstruct( ics, spectrum[0] );
if ( id == 0 && centre < 0 && left < 0 ) centre = outputs;
Synthesize( ics, spectrum[0], outputs++ );
}
else SkipIcs( r );
break;
case 1: // CPE
r.Read( 4 );
DecodeCpe( r, outputs );
if ( left < 0 ) { left = outputs; right = outputs + 1; }
outputs += 2;
break;
case 4: // DSE
{
r.Read( 4 );
var align = r.Read( 1 );
var count = r.Read( 8 );
if ( count == 255 ) count += r.Read( 8 );
if ( align == 1 ) r.ByteAlign();
r.Skip( count * 8 );
break;
}
case 5: // PCE
SkipPce( r );
break;
case 6: // FIL (SBR and other extensions: ignored)
{
var count = r.Read( 4 );
if ( count == 15 ) count += r.Read( 8 ) - 1;
r.Skip( count * 8 );
break;
}
default: // CCE: coupling isn't supported
return null;
}
}
}
catch ( IndexOutOfRangeException )
{
return null; // truncated or corrupt frame
}
if ( outputs == 0 ) return null;
var result = new short[1024 * Channels];
if ( Channels == 1 )
{
var src = pcm[centre >= 0 ? centre : 0];
for ( int i = 0; i < 1024; i++ ) result[i] = Clip( src[i] );
return result;
}
if ( left < 0 )
{
// a mono stream without channel config 1: both sides the same
var src = pcm[centre >= 0 ? centre : 0];
for ( int i = 0; i < 1024; i++ ) result[2 * i] = result[2 * i + 1] = Clip( src[i] );
return result;
}
var l = pcm[left];
var rr = pcm[right];
var c = centre >= 0 ? pcm[centre] : null;
for ( int i = 0; i < 1024; i++ )
{
var cc = c is null ? 0 : c[i] * 0.7071f;
result[2 * i] = Clip( l[i] + cc );
result[2 * i + 1] = Clip( rr[i] + cc );
}
return result;
}
static short Clip( float v ) => (short)Math.Clamp( (int)MathF.Round( v ), short.MinValue, short.MaxValue );
//
// bitstream
//
sealed class Ics
{
public int GlobalGain;
public int WindowSequence, WindowShape, MaxSfb;
public int NumWindows, NumWindowGroups;
public readonly int[] WindowGroupLength = new int[8];
public readonly int[] SfbCb = new int[8 * 64];
public readonly int[] ScaleFactors = new int[8 * 64];
public bool PulsePresent;
public int PulseCount, PulseStartSfb;
public readonly int[] PulseOffset = new int[4], PulseAmp = new int[4];
public bool TnsPresent;
public readonly int[] TnsFilters = new int[8];
public readonly int[] TnsCoefRes = new int[8];
public readonly int[,] TnsLength = new int[8, 4], TnsOrder = new int[8, 4], TnsDirection = new int[8, 4];
public readonly float[,,] TnsLpc = new float[8, 4, 32];
public readonly int[] Quant = new int[1024];
public bool IsLong => WindowSequence != EightShort;
}
void ReadIcsInfo( BitReader r, Ics ics )
{
r.Read( 1 ); // ics_reserved_bit
ics.WindowSequence = r.Read( 2 );
ics.WindowShape = r.Read( 1 );
if ( ics.WindowSequence == EightShort )
{
ics.MaxSfb = r.Read( 4 );
var grouping = r.Read( 7 );
ics.NumWindows = 8;
ics.NumWindowGroups = 1;
ics.WindowGroupLength[0] = 1;
for ( int i = 0; i < 7; i++ )
{
if ( (grouping & (1 << (6 - i))) != 0 ) ics.WindowGroupLength[ics.NumWindowGroups - 1]++;
else ics.WindowGroupLength[ics.NumWindowGroups++] = 1;
}
}
else
{
ics.MaxSfb = r.Read( 6 );
ics.NumWindows = 1;
ics.NumWindowGroups = 1;
ics.WindowGroupLength[0] = 1;
if ( r.Read( 1 ) == 1 ) throw new IndexOutOfRangeException( "prediction" ); // AAC Main only
}
var numSwb = ics.IsLong ? numSwbLong : numSwbShort;
if ( ics.MaxSfb > numSwb ) throw new IndexOutOfRangeException( "max_sfb" );
}
void DecodeIcs( BitReader r, Ics ics, bool commonWindow )
{
ics.GlobalGain = r.Read( 8 );
if ( !commonWindow ) ReadIcsInfo( r, ics );
ReadSections( r, ics );
ReadScaleFactors( r, ics );
ics.PulsePresent = r.Read( 1 ) == 1;
if ( ics.PulsePresent )
{
if ( !ics.IsLong ) throw new IndexOutOfRangeException( "pulse in short window" );
ics.PulseCount = r.Read( 2 ) + 1;
ics.PulseStartSfb = r.Read( 6 );
for ( int i = 0; i < ics.PulseCount; i++ )
{
ics.PulseOffset[i] = r.Read( 5 );
ics.PulseAmp[i] = r.Read( 4 );
}
}
ics.TnsPresent = r.Read( 1 ) == 1;
if ( ics.TnsPresent ) ReadTns( r, ics );
if ( r.Read( 1 ) == 1 ) throw new IndexOutOfRangeException( "gain control" ); // AAC SSR only
ReadSpectrum( r, ics );
}
void ReadSections( BitReader r, Ics ics )
{
var bits = ics.IsLong ? 5 : 3;
var escape = (1 << bits) - 1;
for ( int g = 0; g < ics.NumWindowGroups; g++ )
{
var k = 0;
while ( k < ics.MaxSfb )
{
var cb = r.Read( 4 );
if ( cb == 12 ) throw new IndexOutOfRangeException( "reserved codebook" );
var length = 0;
int incr;
while ( (incr = r.Read( bits )) == escape ) length += escape;
length += incr;
if ( k + length > ics.MaxSfb ) throw new IndexOutOfRangeException( "section" );
for ( int s = k; s < k + length; s++ ) ics.SfbCb[g * 64 + s] = cb;
k += length;
}
}
}
void ReadScaleFactors( BitReader r, Ics ics )
{
var scaleFactor = ics.GlobalGain;
var isPosition = 0;
var noiseEnergy = ics.GlobalGain - 90;
var noisePcm = true;
for ( int g = 0; g < ics.NumWindowGroups; g++ )
{
for ( int sfb = 0; sfb < ics.MaxSfb; sfb++ )
{
var i = g * 64 + sfb;
switch ( ics.SfbCb[i] )
{
case ZeroHcb:
ics.ScaleFactors[i] = 0;
break;
case IntensityHcb:
case IntensityHcb2:
isPosition += Huffman.ScaleFactor.Decode( r ) - 60;
ics.ScaleFactors[i] = isPosition;
break;
case NoiseHcb:
if ( noisePcm ) { noisePcm = false; noiseEnergy += r.Read( 9 ) - 256; }
else noiseEnergy += Huffman.ScaleFactor.Decode( r ) - 60;
ics.ScaleFactors[i] = noiseEnergy;
break;
default:
scaleFactor += Huffman.ScaleFactor.Decode( r ) - 60;
ics.ScaleFactors[i] = scaleFactor;
break;
}
}
}
}
void ReadTns( BitReader r, Ics ics )
{
var isLong = ics.IsLong;
Span<float> parcor = stackalloc float[32];
Span<float> a = stackalloc float[33];
Span<float> b = stackalloc float[33];
for ( int w = 0; w < ics.NumWindows; w++ )
{
var filters = ics.TnsFilters[w] = r.Read( isLong ? 2 : 1 );
if ( filters == 0 ) continue;
var coefRes = ics.TnsCoefRes[w] = r.Read( 1 );
for ( int f = 0; f < filters; f++ )
{
ics.TnsLength[w, f] = r.Read( isLong ? 6 : 4 );
var order = ics.TnsOrder[w, f] = r.Read( isLong ? 5 : 3 );
if ( order == 0 ) continue;
ics.TnsDirection[w, f] = r.Read( 1 );
var compress = r.Read( 1 );
var coefBits = coefRes + 3 - compress;
// the coefficients are quantized reflection coefficients: back to a direct form LPC
var iqfac = ((1 << (coefRes + 3 - 1)) - 0.5f) / (MathF.PI / 2);
var iqfacM = ((1 << (coefRes + 3 - 1)) + 0.5f) / (MathF.PI / 2);
for ( int i = 0; i < order; i++ )
{
var v = r.Read( coefBits );
if ( (v & (1 << (coefBits - 1))) != 0 ) v -= 1 << coefBits;
parcor[i] = MathF.Sin( v / (v >= 0 ? iqfac : iqfacM) );
}
a.Clear();
a[0] = 1;
for ( int m = 1; m <= order; m++ )
{
for ( int i = 1; i < m; i++ ) b[i] = a[i] + parcor[m - 1] * a[m - i];
for ( int i = 1; i < m; i++ ) a[i] = b[i];
a[m] = parcor[m - 1];
}
for ( int i = 0; i <= order; i++ ) ics.TnsLpc[w, f, i] = a[i];
}
}
}
void ReadSpectrum( BitReader r, Ics ics )
{
Array.Clear( ics.Quant );
var swb = ics.IsLong ? swbLong : swbShort;
var groupStart = 0; // first window of the group
for ( int g = 0; g < ics.NumWindowGroups; g++ )
{
var groupLength = ics.WindowGroupLength[g];
for ( int sfb = 0; sfb < ics.MaxSfb; sfb++ )
{
var cb = ics.SfbCb[g * 64 + sfb];
if ( cb is ZeroHcb or NoiseHcb or IntensityHcb or IntensityHcb2 ) continue;
var book = Huffman.Spectral[cb];
var width = swb[sfb + 1] - swb[sfb];
// within a group, a band's coefficients come window by window
for ( int w = 0; w < groupLength; w++ )
{
var at = (groupStart + w) * 128 + swb[sfb];
if ( ics.IsLong ) at = swb[sfb];
for ( int k = 0; k < width; k += book.Dimension )
book.DecodeValues( r, ics.Quant, at + k );
}
}
groupStart += groupLength;
}
if ( ics.PulsePresent )
{
int k = swbLong[Math.Min( ics.PulseStartSfb, numSwbLong )];
for ( int i = 0; i < ics.PulseCount; i++ )
{
k += ics.PulseOffset[i];
if ( k >= 1024 ) break;
ics.Quant[k] += ics.Quant[k] > 0 ? ics.PulseAmp[i] : -ics.PulseAmp[i];
}
}
}
void SkipIcs( BitReader r )
{
// an extra channel we won't play still has to be read through
DecodeIcs( r, icsB, false );
}
void DecodeCpe( BitReader r, int outputs )
{
var a = icsA;
var b = icsB;
var commonWindow = r.Read( 1 ) == 1;
var msMask = 0;
Span<bool> msUsed = stackalloc bool[8 * 64];
if ( commonWindow )
{
ReadIcsInfo( r, a );
CopyInfo( a, b );
msMask = r.Read( 2 );
if ( msMask == 1 )
for ( int g = 0; g < a.NumWindowGroups; g++ )
for ( int sfb = 0; sfb < a.MaxSfb; sfb++ )
msUsed[g * 64 + sfb] = r.Read( 1 ) == 1;
else if ( msMask == 2 ) msUsed.Fill( true );
}
DecodeIcs( r, a, commonWindow );
DecodeIcs( r, b, commonWindow );
if ( outputs + 1 >= 4 ) return;
var l = spectrum[0];
var rr = spectrum[1];
Dequantize( a, l );
Dequantize( b, rr );
// PNS: a band that's noise in both channels with M/S on gets the same noise in both
Noise( a, l, null, Span<bool>.Empty );
Noise( b, rr, commonWindow && msMask > 0 ? a : null, commonWindow ? msUsed : Span<bool>.Empty, l );
if ( commonWindow && msMask > 0 ) MidSide( a, b, l, rr, msUsed );
Intensity( b, l, rr, msUsed, msMask );
Tns( a, l );
Tns( b, rr );
Synthesize( a, l, outputs );
Synthesize( b, rr, outputs + 1 );
}
static void CopyInfo( Ics from, Ics to )
{
to.WindowSequence = from.WindowSequence;
to.WindowShape = from.WindowShape;
to.MaxSfb = from.MaxSfb;
to.NumWindows = from.NumWindows;
to.NumWindowGroups = from.NumWindowGroups;
Array.Copy( from.WindowGroupLength, to.WindowGroupLength, 8 );
}
void SkipPce( BitReader r )
{
r.Read( 4 + 2 + 4 ); // tag, object type, sampling index
int front = r.Read( 4 ), side = r.Read( 4 ), back = r.Read( 4 ), lfe = r.Read( 2 ), assoc = r.Read( 3 ), cc = r.Read( 4 );
if ( r.Read( 1 ) == 1 ) r.Read( 4 ); // mono mixdown
if ( r.Read( 1 ) == 1 ) r.Read( 4 ); // stereo mixdown
if ( r.Read( 1 ) == 1 ) r.Read( 3 ); // matrix mixdown
r.Skip( (front + side + back) * 5 + lfe * 4 + assoc * 4 + cc * 5 );
r.ByteAlign();
r.Skip( r.Read( 8 ) * 8 ); // comment
}
//
// spectral processing
//
/// <summary> Inverse quantization and scaling of one channel into <paramref name="spec"/> (1024 values, window by window). </summary>
void Dequantize( Ics ics, float[] spec )
{
Array.Clear( spec );
var swb = ics.IsLong ? swbLong : swbShort;
var window = 0;
for ( int g = 0; g < ics.NumWindowGroups; g++ )
{
for ( int w = 0; w < ics.WindowGroupLength[g]; w++, window++ )
{
var baseIndex = window * 128;
for ( int sfb = 0; sfb < ics.MaxSfb; sfb++ )
{
var cb = ics.SfbCb[g * 64 + sfb];
if ( cb is ZeroHcb or NoiseHcb or IntensityHcb or IntensityHcb2 ) continue;
var gain = MathF.Pow( 2, 0.25f * (ics.ScaleFactors[g * 64 + sfb] - 100) );
for ( int k = swb[sfb]; k < swb[sfb + 1]; k++ )
{
var q = ics.Quant[baseIndex + k];
if ( q != 0 ) spec[baseIndex + k] = (q > 0 ? Pow43( q ) : -Pow43( -q )) * gain;
}
}
}
}
}
static readonly float[] Pow43Table = MakePow43();
static float[] MakePow43()
{
var t = new float[8192];
for ( int i = 0; i < t.Length; i++ ) t[i] = MathF.Pow( i, 4.0f / 3.0f );
return t;
}
static float Pow43( int q ) => q < 8192 ? Pow43Table[q] : MathF.Pow( q, 4.0f / 3.0f );
/// <summary> Only the single channel path: dequantize, noise, TNS. </summary>
void Reconstruct( Ics ics, float[] spec )
{
Dequantize( ics, spec );
Noise( ics, spec, null, Span<bool>.Empty );
Tns( ics, spec );
}
/// <summary> Perceptual noise substitution: noise bands get random values of the coded energy. </summary>
void Noise( Ics ics, float[] spec, Ics other, ReadOnlySpan<bool> msUsed, float[] otherSpec = null )
{
var swb = ics.IsLong ? swbLong : swbShort;
var window = 0;
for ( int g = 0; g < ics.NumWindowGroups; g++ )
{
for ( int w = 0; w < ics.WindowGroupLength[g]; w++, window++ )
{
var baseIndex = window * 128;
for ( int sfb = 0; sfb < ics.MaxSfb; sfb++ )
{
var i = g * 64 + sfb;
if ( ics.SfbCb[i] != NoiseHcb ) continue;
int from = baseIndex + swb[sfb], to = baseIndex + swb[sfb + 1];
var correlated = other is not null && otherSpec is not null && other.SfbCb[i] == NoiseHcb && msUsed.Length > i && msUsed[i];
float energy = 0;
for ( int k = from; k < to; k++ )
{
var v = correlated ? otherSpec[k] : NextNoise();
spec[k] = v;
energy += v * v;
}
if ( energy <= 0 ) continue;
var scale = MathF.Pow( 2, 0.25f * ics.ScaleFactors[i] ) / MathF.Sqrt( energy );
for ( int k = from; k < to; k++ ) spec[k] *= scale;
}
}
}
}
float NextNoise()
{
noiseState = noiseState * 1664525u + 1013904223u;
return (int)noiseState / 2147483648.0f;
}
void MidSide( Ics ics, Ics right, float[] l, float[] r, ReadOnlySpan<bool> msUsed )
{
var swb = ics.IsLong ? swbLong : swbShort;
var window = 0;
for ( int g = 0; g < ics.NumWindowGroups; g++ )
{
for ( int w = 0; w < ics.WindowGroupLength[g]; w++, window++ )
{
for ( int sfb = 0; sfb < ics.MaxSfb; sfb++ )
{
var i = g * 64 + sfb;
if ( !msUsed[i] || right.SfbCb[i] >= NoiseHcb || ics.SfbCb[i] == NoiseHcb ) continue;
for ( int k = window * 128 + swb[sfb]; k < window * 128 + swb[sfb + 1]; k++ )
{
var m = l[k];
var s = r[k];
l[k] = m + s;
r[k] = m - s;
}
}
}
}
}
void Intensity( Ics right, float[] l, float[] r, ReadOnlySpan<bool> msUsed, int msMask )
{
var swb = right.IsLong ? swbLong : swbShort;
var window = 0;
for ( int g = 0; g < right.NumWindowGroups; g++ )
{
for ( int w = 0; w < right.WindowGroupLength[g]; w++, window++ )
{
for ( int sfb = 0; sfb < right.MaxSfb; sfb++ )
{
var i = g * 64 + sfb;
var cb = right.SfbCb[i];
if ( cb is not (IntensityHcb or IntensityHcb2) ) continue;
var scale = MathF.Pow( 0.5f, 0.25f * right.ScaleFactors[i] );
if ( cb == IntensityHcb2 ) scale = -scale;
if ( msMask > 0 && msUsed[i] ) scale = -scale;
for ( int k = window * 128 + swb[sfb]; k < window * 128 + swb[sfb + 1]; k++ ) r[k] = l[k] * scale;
}
}
}
}
void Tns( Ics ics, float[] spec )
{
if ( !ics.TnsPresent ) return;
var isLong = ics.IsLong;
var swb = isLong ? swbLong : swbShort;
var numSwb = isLong ? numSwbLong : numSwbShort;
var maxBands = Math.Min( (isLong ? AacTables.TnsMaxBandsLong : AacTables.TnsMaxBandsShort)[Math.Min( rateIndex, 11 )], ics.MaxSfb );
Span<float> state = stackalloc float[20];
for ( int w = 0; w < ics.NumWindows; w++ )
{
var bottom = numSwb;
for ( int f = 0; f < ics.TnsFilters[w]; f++ )
{
var top = bottom;
bottom = Math.Max( top - ics.TnsLength[w, f], 0 );
var order = Math.Min( ics.TnsOrder[w, f], isLong ? 20 : 7 );
if ( order == 0 ) continue;
var start = swb[Math.Min( bottom, maxBands )];
var end = swb[Math.Min( top, maxBands )];
var size = end - start;
if ( size <= 0 ) continue;
int at = w * 128 + (ics.TnsDirection[w, f] == 1 ? end - 1 : start);
var step = ics.TnsDirection[w, f] == 1 ? -1 : 1;
state.Clear();
for ( int n = 0; n < size; n++, at += step )
{
// all-pole filter: y = x - sum(a[i] * y[n-i])
var y = spec[at];
for ( int i = 0; i < order; i++ ) y -= state[i] * ics.TnsLpc[w, f, i + 1];
for ( int i = order - 1; i > 0; i-- ) state[i] = state[i - 1];
state[0] = y;
spec[at] = y;
}
}
}
}
//
// filterbank
//
static readonly float[] SineLong = SineWindow( 2048 ), SineShort = SineWindow( 256 );
static readonly float[] KbdLong = KbdWindow( 2048, 4 ), KbdShort = KbdWindow( 256, 6 );
static float[] SineWindow( int n )
{
var w = new float[n / 2];
for ( int i = 0; i < n / 2; i++ ) w[i] = MathF.Sin( MathF.PI / n * (i + 0.5f) );
return w;
}
/// <summary> The rising half of a Kaiser-Bessel-derived window. </summary>
static float[] KbdWindow( int n, double alpha )
{
var half = n / 2;
var kernel = new double[half + 1];
for ( int i = 0; i <= half; i++ )
{
var x = (i - half / 2.0) / (half / 2.0);
kernel[i] = BesselI0( Math.PI * alpha * Math.Sqrt( Math.Max( 0, 1 - x * x ) ) );
}
var total = 0.0;
for ( int i = 0; i <= half; i++ ) total += kernel[i];
var w = new float[half];
var sum = 0.0;
for ( int i = 0; i < half; i++ )
{
sum += kernel[i];
w[i] = (float)Math.Sqrt( sum / total );
}
return w;
}
static double BesselI0( double x )
{
double sum = 1, term = 1;
for ( int k = 1; k < 50; k++ )
{
term *= (x / (2 * k)) * (x / (2 * k));
sum += term;
if ( term < 1e-12 * sum ) break;
}
return sum;
}
static float[] Rising( int shape, bool isLong ) => shape == 1 ? (isLong ? KbdLong : KbdShort) : (isLong ? SineLong : SineShort);
/// <summary> IMDCT, window, overlap-add: 1024 new samples for output channel <paramref name="ch"/>. </summary>
void Synthesize( Ics ics, float[] spec, int ch )
{
if ( ch >= 4 ) return;
var t = timeBuffer;
var prevShape = previousShape[ch];
var shape = ics.WindowShape;
var longLeft = Rising( prevShape, true );
var longRight = Rising( shape, true );
var shortLeft = Rising( prevShape, false );
var shortRight = Rising( shape, false );
if ( ics.WindowSequence == EightShort )
{
Array.Clear( t );
Span<float> block = stackalloc float[256];
for ( int w = 0; w < 8; w++ )
{
Imdct.Short.Inverse( spec.AsSpan( w * 128, 128 ), block );
var left = w == 0 ? shortLeft : shortRight;
var at = 448 + w * 128;
for ( int i = 0; i < 128; i++ )
{
t[at + i] += block[i] * left[i];
t[at + 128 + i] += block[128 + i] * shortRight[127 - i];
}
}
}
else
{
Imdct.Long.Inverse( spec.AsSpan( 0, 1024 ), t );
switch ( ics.WindowSequence )
{
case OnlyLong:
for ( int i = 0; i < 1024; i++ ) { t[i] *= longLeft[i]; t[1024 + i] *= longRight[1023 - i]; }
break;
case LongStart:
for ( int i = 0; i < 1024; i++ ) t[i] *= longLeft[i];
for ( int i = 0; i < 128; i++ ) t[1472 + i] *= shortRight[127 - i];
for ( int i = 1600; i < 2048; i++ ) t[i] = 0;
break;
case LongStop:
for ( int i = 0; i < 448; i++ ) t[i] = 0;
for ( int i = 0; i < 128; i++ ) t[448 + i] *= shortLeft[i];
for ( int i = 0; i < 1024; i++ ) t[1024 + i] *= longRight[1023 - i];
break;
}
}
var o = overlap[ch];
var output = pcm[ch];
for ( int i = 0; i < 1024; i++ )
{
output[i] = t[i] + o[i];
o[i] = t[1024 + i];
}
previousShape[ch] = shape;
}
/// <summary> Inverse MDCT through a DCT-IV of half the length, done with a complex FFT of a quarter. </summary>
sealed class Imdct
{
public static readonly Imdct Long = new( 2048 ), Short = new( 256 );
readonly int n, m; // window length, coefficient count (n / 2)
readonly float[] twiddleCos, twiddleSin; // exp(-i pi (4k+1) / 4m)
readonly Fft fft;
readonly float[] re, im, dct;
Imdct( int length )
{
n = length;
m = length / 2;
twiddleCos = new float[m / 2];
twiddleSin = new float[m / 2];
for ( int k = 0; k < m / 2; k++ )
{
var a = -Math.PI * (4 * k + 1) / (4.0 * m);
twiddleCos[k] = (float)Math.Cos( a );
twiddleSin[k] = (float)Math.Sin( a );
}
fft = new Fft( m / 2 );
re = new float[m / 2];
im = new float[m / 2];
dct = new float[m];
}
/// <summary> <paramref name="spec"/>: m coefficients. <paramref name="output"/>: n samples, scaled by 2/n. </summary>
public void Inverse( ReadOnlySpan<float> spec, Span<float> output )
{
var half = m / 2;
// DCT-IV of spec into dct[]
for ( int k = 0; k < half; k++ )
{
float xr = spec[2 * k], xi = spec[m - 1 - 2 * k];
re[k] = xr * twiddleCos[k] - xi * twiddleSin[k];
im[k] = xr * twiddleSin[k] + xi * twiddleCos[k];
}
fft.Forward( re, im );
for ( int k = 0; k < half; k++ )
{
float yr = re[k] * twiddleCos[k] - im[k] * twiddleSin[k];
float yi = re[k] * twiddleSin[k] + im[k] * twiddleCos[k];
dct[2 * k] = yr;
dct[m - 1 - 2 * k] = -yi;
}
// unfold to the IMDCT's n outputs (phase n0 = m/2 + 1/2)
var scale = 2.0f / n;
for ( int i = 0; i < half; i++ ) output[i] = dct[i + half] * scale;
for ( int i = half; i < 3 * half; i++ ) output[i] = -dct[3 * half - 1 - i] * scale;
for ( int i = 3 * half; i < n; i++ ) output[i] = -dct[i - 3 * half] * scale;
}
}
/// <summary> In-place radix-2 complex FFT (forward, e^-i). </summary>
sealed class Fft
{
readonly int size;
readonly int[] reverse;
readonly float[] cos, sin;
public Fft( int size )
{
this.size = size;
reverse = new int[size];
var bits = 0;
while ( (1 << bits) < size ) bits++;
for ( int i = 0; i < size; i++ )
{
var r = 0;
for ( int b = 0; b < bits; b++ ) if ( (i & (1 << b)) != 0 ) r |= 1 << (bits - 1 - b);
reverse[i] = r;
}
cos = new float[size / 2];
sin = new float[size / 2];
for ( int i = 0; i < size / 2; i++ )
{
cos[i] = (float)Math.Cos( -2 * Math.PI * i / size );
sin[i] = (float)Math.Sin( -2 * Math.PI * i / size );
}
}
public void Forward( float[] re, float[] im )
{
for ( int i = 0; i < size; i++ )
{
var j = reverse[i];
if ( j <= i ) continue;
(re[i], re[j]) = (re[j], re[i]);
(im[i], im[j]) = (im[j], im[i]);
}
for ( int len = 2; len <= size; len <<= 1 )
{
var halfLen = len / 2;
var step = size / len;
for ( int i = 0; i < size; i += len )
{
for ( int j = 0; j < halfLen; j++ )
{
float wr = cos[j * step], wi = sin[j * step];
int a = i + j, b = a + halfLen;
float tr = re[b] * wr - im[b] * wi;
float ti = re[b] * wi + im[b] * wr;
re[b] = re[a] - tr;
im[b] = im[a] - ti;
re[a] += tr;
im[a] += ti;
}
}
}
}
}
//
// Huffman
//
/// <summary> A codebook as a binary tree, built once from the code / length tables. </summary>
sealed class Huffman
{
public static readonly Huffman ScaleFactor = new( AacTables.CodeScf, AacTables.LengthScf, 1, 0, false, false );
public static readonly Huffman[] Spectral =
{
null,
new( AacTables.Code1, AacTables.Length1, 4, 3, true, false ),
new( AacTables.Code2, AacTables.Length2, 4, 3, true, false ),
new( AacTables.Code3, AacTables.Length3, 4, 3, false, false ),
new( AacTables.Code4, AacTables.Length4, 4, 3, false, false ),
new( AacTables.Code5, AacTables.Length5, 2, 9, true, false ),
new( AacTables.Code6, AacTables.Length6, 2, 9, true, false ),
new( AacTables.Code7, AacTables.Length7, 2, 8, false, false ),
new( AacTables.Code8, AacTables.Length8, 2, 8, false, false ),
new( AacTables.Code9, AacTables.Length9, 2, 13, false, false ),
new( AacTables.Code10, AacTables.Length10, 2, 13, false, false ),
new( AacTables.Code11, AacTables.Length11, 2, 17, false, true ),
};
public readonly int Dimension;
readonly int modulo;
readonly bool signed, escape;
readonly int[] tree; // node * 2 + bit: > 0 next node, <= 0 leaf -(index)
Huffman( uint[] codes, byte[] lengths, int dimension, int modulo, bool signed, bool escape )
{
Dimension = dimension;
this.modulo = modulo;
this.signed = signed;
this.escape = escape;
var nodes = new List<int> { 0, 0 };
var used = new List<bool> { false, false };
for ( int index = 0; index < codes.Length; index++ )
{
var node = 0;
for ( int b = lengths[index] - 1; b >= 0; b-- )
{
var bit = (int)(codes[index] >> b) & 1;
var slot = node * 2 + bit;
if ( b == 0 )
{
nodes[slot] = -index;
used[slot] = true;
break;
}
if ( !used[slot] )
{
used[slot] = true;
nodes[slot] = nodes.Count / 2;
nodes.Add( 0 ); nodes.Add( 0 );
used.Add( false ); used.Add( false );
}
node = nodes[slot];
}
}
tree = nodes.ToArray();
}
public int Decode( BitReader r )
{
var node = 0;
while ( true )
{
var next = tree[node * 2 + r.Read( 1 )];
if ( next <= 0 ) return -next;
node = next;
}
}
/// <summary> One codeword's values (4 or 2), with their signs and escapes, into <paramref name="dest"/>. </summary>
public void DecodeValues( BitReader r, int[] dest, int at )
{
var index = Decode( r );
Span<int> v = stackalloc int[4];
if ( Dimension == 4 )
{
v[0] = index / 27; v[1] = index / 9 % 3; v[2] = index / 3 % 3; v[3] = index % 3;
}
else
{
v[0] = index / modulo; v[1] = index % modulo;
}
if ( signed )
{
var offset = Dimension == 4 ? 1 : 4;
for ( int i = 0; i < Dimension; i++ ) v[i] -= offset;
}
else
{
for ( int i = 0; i < Dimension; i++ )
if ( v[i] != 0 && r.Read( 1 ) == 1 ) v[i] = -v[i];
}
if ( escape )
{
for ( int i = 0; i < 2; i++ )
{
if ( Math.Abs( v[i] ) != 16 ) continue;
var n = 4;
while ( r.Read( 1 ) == 1 ) { n++; if ( n > 12 ) throw new IndexOutOfRangeException( "escape" ); }
var value = (1 << n) + r.Read( n );
v[i] = v[i] < 0 ? -value : value;
}
}
for ( int i = 0; i < Dimension && at + i < dest.Length; i++ ) dest[at + i] = v[i];
}
}
sealed class BitReader
{
readonly byte[] data;
int bit;
public BitReader( byte[] data ) { this.data = data; }
public int BitsLeft => data.Length * 8 - bit;
public int Read( int n )
{
if ( n == 0 ) return 0;
if ( bit + n > data.Length * 8 ) throw new IndexOutOfRangeException( "end of frame" );
var v = 0;
for ( int i = 0; i < n; i++, bit++ )
v = (v << 1) | ((data[bit >> 3] >> (7 - (bit & 7))) & 1);
return v;
}
public void Skip( int n )
{
if ( bit + n > data.Length * 8 ) throw new IndexOutOfRangeException( "end of frame" );
bit += n;
}
public void ByteAlign() => bit = (bit + 7) & ~7;
}
}
Game
library
namespace Bimp.Resolver.Media;
/// <summary>
/// A source of finished segment files for <see cref="SegmentPlayer"/>: merged YouTube media
/// (<see cref="WebmSegmenter"/>) or a live stream (<see cref="Live.LiveSegmenter"/>).
/// </summary>
public interface ISegmentFeed
{
/// <summary> Live: segments appear as the stream comes in, old ones go away, there's no end to seek in. </summary>
bool IsLive { get; }
/// <summary>
/// Live: how much stream time must be buffered past the start point before playback starts, so the next
/// segment is always finished before the current one ends. Sources that deliver in chunks (HLS) need more.
/// </summary>
double StartBufferSeconds => 0;
/// <summary>
/// Live, low latency: short segments that end in filler frames instead of an overlap - join the newest one and
/// keep close behind the live edge.
/// </summary>
bool LowLatency => false;
/// <summary> The newest finished segment, -1 if none yet. </summary>
int Newest { get; }
/// <summary> The segment if it's finished (and still on disk), else null. </summary>
MediaSegment TryGet( int seq );
/// <summary> Playback reached this segment - prepare it and what follows. </summary>
void Want( int seq );
/// <summary> Done with a segment - its file can go. </summary>
void Release( int seq );
/// <summary> Something went wrong, user facing. </summary>
string Error { get; }
}
Game
library
using Bimp.Resolver.Media;
namespace Bimp;
/// <summary>
/// Plays a series of local segment files back to back: merged YouTube media (<see cref="WebmSegmenter"/>) or a
/// live stream remuxed to MP4 (<see cref="Resolver.Live.LiveSegmenter"/>).
/// <para>
/// Segment N plays while N+1 is started just in time in a second, hidden and silent player; at the boundary
/// they swap, so the picture carries straight on. Segments keep absolute timestamps, so <see cref="Time"/>
/// is the media time.
/// </para>
/// </summary>
public sealed class SegmentPlayer : IDisposable
{
class Slot
{
public MediaSegment Segment;
public VideoPlayer Player;
public bool Loaded;
public bool Finished;
public bool Preload; // waiting in the wings: silent until it's swapped in
public bool FirstFrame; // set when it delivers its first frame (from OnTextureData, during Present)
public bool Primed; // its decoder has produced its first frame
public int Steady; // frames in a row it has delivered on time (see SettleFrames)
public RealTimeSince SinceFrame;
public bool Fading; // swapped out: silent (its volume ramps down) and closed a moment later
public float LastTime;
public RealTimeSince SinceOpened;
}
readonly ISegmentFeed feed;
/// <summary>
/// Live: start this many segments behind the newest finished one, so the next is always ready by the time the
/// current one ends. Latency is roughly (JoinBehind + 1) segments plus the overlap.
/// </summary>
int JoinBehind => feed.LowLatency ? 0 : 1;
/// <summary> Live: further behind than this (slow machine, stalled download) jumps back to the live edge. </summary>
int MaxBehind => feed.LowLatency ? 1 : 3;
/// <summary>
/// Low latency: join the newest segment this long after it's written, so the next one is ready a little before
/// the current one ends (its start latency plus arrival jitter). A segment that's late anyway just holds the
/// last picture (the filler frames) until it arrives.
/// </summary>
const double LowJoinMargin = 0.2;
/// <summary>
/// Average time from opening a segment to its first decoded frame (measured as we go).
/// </summary>
float startLatency;
/// <summary>
/// How long before the current segment ends to start the next one, so its first frame lands on the boundary -
/// and so the two players are close to in step for the audio crossfade. A player that isn't primed in time
/// just swaps a moment later (the current one plays on into its overlap, if the file has one).
/// </summary>
float StartLead => JustInTime
? Math.Clamp( (startLatency > 0 ? startLatency : 0.08f) + 0.015f, 0.0f, 1.0f )
: ClockLeadOnly ? Math.Clamp( clockLead, 0.0f, Math.Min( current.Segment.Overlap, 1.0f ) )
: Math.Clamp( (startLatency > 0 ? startLatency : 0.04f) + leadCorrection, 0.0f, feed.IsLive ? 1.0f : 3.0f );
/// <summary>
/// Media files that run on into the next segment (<see cref="MediaSegment.Overlap"/>): the current player shows real
/// frames until the next one's first frame is up, however long that takes (150-520 ms at 1440p/4K), so the next one
/// only has to be opened for its clock to be in step. Adding the start latency to that made it skip that far ahead
/// at a swap, and correcting that back made the current one hold its last frame instead.
/// <para>
/// Merged WebM (VP9/AV1) only: MP4 segments (HLS VOD) kept waiting for the next H.264 player to settle made Media
/// Foundation drop a sample ("MFT deadlock") in every run, and the picture broke up until the next keyframe (3 of 3
/// runs, against 0 of 3 with the old timing).
/// </para>
/// </summary>
bool ClockLeadOnly => feed is WebmSegmenter && (current?.Segment.Overlap ?? 0) > 0;
/// <summary> How long before the end to open the next player so its clock is in step (<see cref="ClockLeadOnly"/>), learned from each swap. </summary>
float clockLead = 0.05f;
/// <summary>
/// <see cref="ClockLeadOnly"/>: swap once the next player has delivered this many frames in a row on time. Its first
/// frame comes late, and a 1440p decoder is still catching up to its clock for a moment after it - swapping on the
/// first frame put that on screen as a 80-120 ms gap (measured). The current one plays on into its overlap meanwhile.
/// </summary>
const int SettleFrames = 4;
const float SettleInterval = 0.045f;
/// <summary>
/// Low latency files that end at their boundary (H.264, ending in skip frames): the next one just opens early enough
/// for its first frame to be ready there. (Syncing clocks made every swap ~40 ms late: a new player's clock runs from
/// Play(), its first frame comes ~60 ms later.)
/// </summary>
bool JustInTime => feed.LowLatency && (current?.Segment.Overlap ?? 0) <= 0;
/// <summary>
/// Learned from each swap: how far the next player's clock was ahead of the current one's at the boundary.
/// Starting it that much later next time brings them in step, which the audio crossfade needs - and so the next
/// player neither skips its first frames nor makes the current one hold its last, while the current one plays on
/// into its overlap (see <see cref="MediaSegment.Overlap"/>).
/// </summary>
float leadCorrection;
/// <summary>
/// How long the swapped-out player keeps running, silent, after a swap.
/// <para>
/// The engine's mixer runs every sound through Steam Audio's direct effect, whose gain interpolator ramps any
/// volume change over the next mix frames. Unmuting the next player while closing the current one outright
/// left a ~70 ms dip in the audio at every boundary (measured with a sine: down to 5%, every segment).
/// Muting the old player at the same moment instead - both ramps at once - makes that a crossfade.
/// </para>
/// </summary>
const float FadeSeconds = 0.15f;
Slot fading;
RealTimeSince sinceFade;
Slot current, next;
bool paused;
// audio settings, applied to both players
Vector3 audioPosition;
bool audioSpatial;
float audioVolume = 1;
float audioDistance = 2500;
string error;
public string Error => error ?? feed.Error;
/// <summary> The segment on screen, -1 before the first. </summary>
public int CurrentSeq => current?.Segment.Seq ?? -1;
/// <summary> Has the first segment started? </summary>
public bool Loaded => current is { Loaded: true } || (Mjpeg?.HasShown ?? false);
/// <summary> Motion JPEG: frames go straight to the texture, there are no segments to play. </summary>
Resolver.Live.LiveMjpeg Mjpeg => (feed as Resolver.Live.LiveSegmenter)?.Mjpeg;
/// <summary> The last segment has played to its end. </summary>
public bool Finished { get; private set; }
/// <summary> Media time (VOD), or running stream time (live). </summary>
public float Time => (float)StreamTime;
/// <summary>
/// Stream time (seconds) of what's on screen. WebM segments keep absolute timestamps (their StreamStart is 0); MP4
/// segments (live, HLS VOD) each start their clock at 0.
/// </summary>
public double StreamTime => Mjpeg is { } mjpeg ? mjpeg.ShownTime : (current?.Segment.StreamStart ?? 0) + (current?.Player?.PlaybackTime ?? 0);
public float Duration => current?.Player?.Duration ?? 0;
public bool IsPaused => current?.Player?.IsPaused ?? paused;
/// <summary> Every segment's frames go to one texture (see <see cref="VideoFrameSink"/>). </summary>
readonly VideoFrameSink sink = new();
public Texture Texture => sink.Texture;
public int Width => sink.Width;
public int Height => sink.Height;
public bool HasVideo => sink.Texture is not null && Width > 4 && Height > 4;
public (int rate, int channels) AudioFormat => (current?.Player?.SampleRate ?? 0, current?.Player?.Channels ?? 0);
/// <summary>
/// Merged media: the synced media time, null while paused. A first segment that starts later than it (at the next
/// keyframe - see <see cref="MediaBackend"/>) opens only when the clock gets there, so it never has to seek forward.
/// </summary>
public float? SyncedTime { get; set; }
public SegmentPlayer( ISegmentFeed feed )
{
this.feed = feed;
if ( !feed.IsLive ) feed.Want( 0 );
}
/// <summary>
/// Call every frame.
/// </summary>
public void Update()
{
if ( Mjpeg is { } mjpeg )
{
if ( !paused ) mjpeg.Present( sink );
return;
}
if ( current is null )
{
var first = feed.IsLive ? LiveJoinSegment() : feed.TryGet( 0 );
if ( first is null ) return;
if ( !feed.IsLive && !paused && SyncedTime is { } now && first.StreamStart + first.Start - now > StartLead ) return;
current = Open( first );
if ( paused ) current.Player?.Pause();
}
// Live and fell far behind? Jump to the live edge.
if ( feed.IsLive && feed.Newest - current.Segment.Seq > MaxBehind && LiveJoinSegment() is { } edge )
{
Log.Info( $"[bimp] live: {feed.Newest - current.Segment.Seq} segments behind, jumping to the live edge" );
Close( next ); next = null;
Close( current );
current = Open( edge );
}
TimedPresent( current, "current" );
// the first segment's own start seeds the start latency, so the first swap isn't late on a slow (4K) decoder
if ( startLatency <= 0 && !current.Primed && current.FirstFrame )
{
current.Primed = true;
startLatency = (float)current.SinceOpened;
}
// the swapped-out player finishes its fade, then goes
if ( fading is not null )
{
TimedPresent( fading, "fading" );
if ( sinceFade > FadeSeconds ) CloseFading();
}
var p = current.Player;
var seg = current.Segment;
// A player started fresh delivers its first frame ~0.25s after Play(), and one that was paused takes ~1s
// after Resume() - so start the next segment just in time (hidden, silent, presented so it runs),
// timed from the measured start latency, and swap which player is on screen. Nothing ever pauses.
if ( next is null && !paused && !seg.Last && current.Loaded && p is not null && p.PlaybackTime >= seg.End - StartLead )
{
var upcoming = feed.TryGet( seg.Seq + 1 );
if ( upcoming is not null ) next = Open( upcoming, preload: true );
}
if ( next?.Player is not null )
{
TimedPresent( next, "next" );
if ( !next.Primed && next.FirstFrame )
{
next.Primed = true;
var latency = (float)next.SinceOpened;
startLatency = startLatency <= 0 ? latency : startLatency * 0.7f + latency * 0.3f;
MediaProbe.Note( $"seg {next.Segment.Seq} first frame {latency * 1000:0}ms after opening (avg {startLatency * 1000:0}ms), current at {p?.PlaybackTime:0.000}/{seg.End:0.000}" );
}
}
// Swap when the current segment's clock reaches its end. (OnFinished fires ~80ms early, while
// frames are still queued - swapping on it cut off the last frames.)
var stuck = current.Finished && p is not null && MathF.Abs( p.PlaybackTime - current.LastTime ) < 0.0001f;
current.LastTime = p?.PlaybackTime ?? 0;
var atEnd = (current.Loaded && p is not null && p.PlaybackTime >= seg.End - 0.005f) || stuck;
if ( !atEnd ) return;
if ( seg.Last )
{
Finished = current.Finished || stuck;
return;
}
// a next player still catching up waits, while the current one has overlap left to play
var settled = !ClockLeadOnly || next is null || next.Steady >= SettleFrames || (p is not null && p.PlaybackTime >= seg.End + seg.Overlap - 0.1f) || stuck;
if ( next is { Primed: true } && settled )
{
// the next player's clock vs the current one's, in stream time: + means it's ahead. (Merged media keeps
// absolute timestamps; live segments each start their clock at 0.)
var ahead = (float)((next.Segment.StreamStart + (next.Player?.PlaybackTime ?? 0)) - (seg.StreamStart + (p?.PlaybackTime ?? seg.End)));
// keep the two clocks in step (see leadCorrection)
if ( ClockLeadOnly ) clockLead = Math.Clamp( clockLead - ahead * 0.7f, 0.0f, 1.0f );
else if ( !JustInTime ) leadCorrection = Math.Clamp( leadCorrection - ahead * 0.7f, -0.2f, 0.2f );
MediaProbe.Note( $"swap {seg.Seq}->{next.Segment.Seq} at {p?.PlaybackTime:0.000}/{seg.End:0.000} (overlap {seg.Overlap * 1000:0}ms), next ahead by {ahead * 1000:0}ms, lead now {StartLead * 1000:0}ms" );
next.Preload = false;
ApplyAudio( next );
// its frame from just before the boundary goes up now, not 33 ms later with its next one
sink.ShowHeld( next.Player );
// crossfade: the old player goes silent as the new one comes up (see FadeSeconds)
CloseFading();
fading = current;
fading.Fading = true;
ApplyAudio( fading );
sinceFade = 0;
current = next;
next = null;
feed.Want( current.Segment.Seq ); // keeps the one after it coming
}
// otherwise the next segment isn't downloaded yet - hold the last frame
}
void TimedPresent( Slot slot, string role )
{
if ( slot?.Player is null ) return;
var t = System.Diagnostics.Stopwatch.GetTimestamp();
slot.Player.Present();
MediaProbe.PresentTook( System.Diagnostics.Stopwatch.GetElapsedTime( t ).TotalMilliseconds, role, slot.Segment.Seq );
}
/// <summary>
/// Live: the segment to start from - the newest one that's at least <see cref="JoinBehind"/> behind the newest
/// finished segment and leaves the feed's start buffer of stream time after it. Null while there isn't enough yet.
/// </summary>
MediaSegment LiveJoinSegment()
{
var newest = feed.TryGet( feed.Newest );
if ( newest is null ) return null;
var bufferedUntil = newest.StreamStart + newest.End;
if ( feed.LowLatency ) return RealTime.NowDouble - newest.ReadyAt >= LowJoinMargin ? newest : null;
for ( var seq = newest.Seq - JoinBehind; seq >= 0; seq-- )
{
var s = feed.TryGet( seq );
if ( s is null ) return null; // not enough on disk yet
if ( bufferedUntil - s.StreamStart >= feed.StartBufferSeconds ) return s;
}
return null;
}
Slot Open( MediaSegment segment, bool preload = false )
{
MediaProbe.Note( $"open seg {segment.Seq} ({segment.Start:0.00}-{segment.End:0.00}){(preload ? " (preload)" : "")}" );
var slot = new Slot { Segment = segment, SinceOpened = 0, Preload = preload };
try
{
var player = new VideoPlayer();
player.OnLoaded += () => slot.Loaded = true;
player.OnFinished += () => slot.Finished = true;
slot.Player = player;
ApplyAudio( slot );
// only the player on screen updates the texture; a preloading one just says it's ready
sink.Attach( player, () => !slot.Preload && !slot.Fading, () =>
{
slot.Steady = slot.FirstFrame && slot.SinceFrame < SettleInterval ? slot.Steady + 1 : 0;
slot.SinceFrame = 0;
slot.FirstFrame = true;
}, $"seg {segment.Seq}", holdWhileInactive: preload );
var playTimer = System.Diagnostics.Stopwatch.StartNew();
player.Play( FileSystem.Data, segment.Path );
MediaProbe.Note( $"Play() seg {segment.Seq} took {playTimer.Elapsed.TotalMilliseconds:0.0}ms" );
}
catch ( Exception e )
{
error = e.Message;
}
return slot;
}
void Close( Slot slot )
{
if ( slot?.Player is null ) return;
var timer = System.Diagnostics.Stopwatch.StartNew();
sink.Detach( slot.Player );
slot.Player.Dispose();
MediaProbe.Note( $"close seg {slot.Segment.Seq} took {timer.Elapsed.TotalMilliseconds:0.0}ms" );
}
void CloseFading()
{
if ( fading is null ) return;
Close( fading );
feed.Release( fading.Segment.Seq );
fading = null;
}
public void SetPaused( bool pause )
{
paused = pause;
if ( current?.Player is { } p )
{
if ( pause ) p.Pause(); else p.Resume();
}
if ( pause ) CloseFading();
// a preloading player would carry on running - start it again when we resume
if ( pause && next is not null )
{
Close( next );
next = null;
}
}
/// <summary>
/// A short hop within the current segment (landing exactly on the synced time after opening).
/// Anything further is done by opening a new segmenter at the time.
/// </summary>
public void Seek( float time )
{
var seg = current?.Segment;
if ( seg is null ) return;
current.Player?.Seek( Math.Clamp( time - (float)seg.StreamStart, seg.Start, seg.End ) );
Finished = false;
}
public void SetAudio( Vector3 position, bool spatial, float volume, float distance )
{
audioPosition = position;
audioSpatial = spatial;
audioVolume = volume;
audioDistance = distance;
ApplyAudio( current );
ApplyAudio( next );
}
void ApplyAudio( Slot slot )
{
if ( slot?.Player is null ) return;
var a = slot.Player.Audio;
a.ListenLocal = !audioSpatial;
a.Position = audioSpatial ? audioPosition : Vector3.Forward * 64.0f;
a.Volume = slot.Preload || slot.Fading ? 0 : audioVolume;
a.Distance = audioDistance;
a.Falloff = MediaPlayer.AudioFalloff;
}
public void Dispose()
{
CloseFading();
Close( current );
Close( next );
current = next = null;
sink.Dispose();
}
}
Game
library
namespace Bimp.Resolver.Extractors;
/// <summary>
/// Every site the native resolver understands.
/// </summary>
public static class ExtractorRegistry
{
// built on each call, not kept in a static: a hotload keeps statics, so new extractors never showed up (their caches
// are statics of their own)
public static IReadOnlyList<IExtractor> All => new IExtractor[]
{
new YouTubeExtractor(),
new StreamableExtractor(),
new VimeoExtractor(),
new SoundCloudExtractor(),
new TwitchExtractor(),
new KickExtractor(),
new XExtractor(),
new ArchiveExtractor(),
new BandcampExtractor(),
new DailymotionExtractor(),
new HlsExtractor(), // any .m3u8 link
new DashExtractor(), // any .mpd link
new LabExtractor(), // [probe] loopback test media
};
public static IExtractor For( Uri url ) => All.FirstOrDefault( e => e.CanHandle( url ) );
public static IExtractor ByKey( string key ) => All.FirstOrDefault( e => e.Key == key );
}
Game
library
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Bimp.Resolver;
namespace Bimp;
/// <summary>
/// A networked media player. The host owns the state (what's playing, when it started, paused, queue)
/// and every client plays it locally, keeping itself in sync with the host's clock.
/// <para>
/// Put this on a GameObject with a <see cref="WorldPanel"/> + <see cref="MediaScreen"/> for a TV/screen,
/// or with <see cref="MediaSpeaker"/> and <see cref="AudioOnlyPlayer"/> for a speaker. Add a collider and
/// <see cref="MediaInteract"/> so players can open the controls with the use key.
/// </para>
/// </summary>
[Title( "Media Player" ), Category( "Media" ), Icon( "smart_display" )]
public sealed class MediaPlayer : Component
{
//
// Inspector settings
//
/// <summary>
/// Who can control playback. The host can always control it.
/// </summary>
[Property, Group( "Access" )] public MediaPermission Permission { get; set; } = MediaPermission.Anyone;
/// <summary>
/// SteamIds allowed to control this player when <see cref="Permission"/> is Whitelist.
/// </summary>
[Property, Group( "Access" ), ShowIf( nameof( Permission ), MediaPermission.Whitelist )]
public List<long> AllowedSteamIds { get; set; } = new();
/// <summary>
/// Let anyone add to the queue, even if they can't control playback.
/// </summary>
[Property, Group( "Access" )] public bool AnyoneCanQueue { get; set; } = true;
[Property, Group( "Queue" ), Range( 1, 200 )] public int MaxQueue { get; set; } = 30;
/// <summary>
/// Replay the current item when it ends and the queue is empty.
/// </summary>
[Property, Group( "Queue" )] public bool Loop { get; set; }
/// <summary>
/// Played by the host when the scene starts.
/// </summary>
[Property, Group( "Playback" )] public string DefaultUrl { get; set; }
/// <summary>
/// Never show video, only play audio. For speakers, radios and jukeboxes.
/// </summary>
[Property, Group( "Playback" )] public bool AudioOnlyPlayer { get; set; }
[Property, Group( "Playback" ), Range( 0, 1 )] public float Volume { get; set; } = 1.0f;
/// <summary>
/// Sound comes from this object's position in the world and falls off with distance.
/// When off, everyone hears it at the same volume wherever they are.
/// </summary>
[Property, Group( "Audio" )] public bool Spatial { get; set; } = true;
/// <summary>
/// Distance at which the sound is fully faded out (see <see cref="AudioFalloff"/>).
/// </summary>
[Property, Group( "Audio" ), ShowIf( nameof( Spatial ), true )] public float AudioDistance { get; set; } = 4000.0f;
/// <summary>
/// How the sound fades out to <see cref="AudioDistance"/>: nearly full volume in the room with the screen, then a
/// smooth fade. The engine's own curve is made for its 15000 unit default range; over a screen's few thousand units
/// it was down to 22% three metres away and 4% at twelve.
/// </summary>
public static readonly Curve AudioFalloff = new( new( 0, 1 ), new( 0.15f, 0.92f ), new( 0.45f, 0.55f ), new( 0.75f, 0.2f ), new( 1, 0 ) );
/// <summary> 3D for this player: the player's own setting, unless they've turned 3D audio off (bimp_spatial). </summary>
public bool EffectiveSpatial => Spatial && MediaSettings.SpatialAudio;
/// <summary>
/// 2D for this player (bimp_spatial off, on a player that's otherwise 3D): the sound has no direction, but still
/// fades with the listener's distance from it, on the same curve as 3D. A player that isn't <see cref="Spatial"/>
/// at all is the same volume everywhere.
/// </summary>
bool FlatWithDistance => Spatial && !MediaSettings.SpatialAudio;
/// <summary> How loud this player is at a point, from <see cref="AudioFalloff"/> over <see cref="AudioDistance"/>. </summary>
float DistanceFade( Vector3 listener )
{
var range = ScaledAudioDistance;
if ( range <= 0 ) return 1;
return AudioFalloff.Evaluate( (listener.Distance( SoundPositionFor( listener ) ) / range).Clamp( 0, 1 ) );
}
/// <summary>
/// Where the sound comes from. Defaults to this GameObject.
/// </summary>
[Property, Group( "Audio" )] public GameObject SoundOrigin { get; set; }
//
// Synced state - only the host writes these
//
/// <summary> The url the user entered. </summary>
[Sync( SyncFlags.FromHost )] public string CurrentUrl { get; set; }
/// <summary> The url clients actually play. Empty when nothing is playing. </summary>
[Sync( SyncFlags.FromHost )] public string PlayUrl { get; set; }
[Sync( SyncFlags.FromHost )] public string Title { get; set; }
[Sync( SyncFlags.FromHost )] public string RequestedBy { get; set; }
/// <summary> Media length in seconds, 0 if unknown or live. </summary>
[Sync( SyncFlags.FromHost )] public float MediaDuration { get; set; }
[Sync( SyncFlags.FromHost )] public bool IsLive { get; set; }
[Sync( SyncFlags.FromHost )] public bool AudioOnly { get; set; }
[Sync( SyncFlags.FromHost )] public bool SeekByReload { get; set; }
/// <summary> Host time (<see cref="Time.NowDouble"/>) at which media time 0 would have played. </summary>
[Sync( SyncFlags.FromHost )] public double StartTime { get; set; }
[Sync( SyncFlags.FromHost )] public bool Paused { get; set; }
[Sync( SyncFlags.FromHost )] public float PausedAt { get; set; }
/// <summary> Increments every time something (re)starts, so replaying the same url reloads it. </summary>
[Sync( SyncFlags.FromHost )] public int PlayId { get; set; }
/// <summary> Increments on every explicit seek, so clients apply it right away rather than treating it as drift. </summary>
[Sync( SyncFlags.FromHost )] public int SeekId { get; set; }
/// <summary> Host status message: "Loading..." or an error. </summary>
[Sync( SyncFlags.FromHost )] public string Status { get; set; }
[Sync( SyncFlags.FromHost )] public NetList<MediaQueueItem> Queue { get; set; } = new();
/// <summary> Video heights the current media is available in (see <see cref="MediaQueueItem.Qualities"/>). </summary>
[Sync( SyncFlags.FromHost )] public string Qualities { get; set; }
/// <summary> Audio tracks of the current media (see <see cref="MediaQueueItem.AudioTracks"/>). </summary>
[Sync( SyncFlags.FromHost )] public string AudioTracks { get; set; }
/// <summary> Caption tracks of the current media (see <see cref="MediaQueueItem.CaptionTracks"/>). </summary>
[Sync( SyncFlags.FromHost )] public string CaptionTracks { get; set; }
//
// Per-client stream choices. Everyone shares the timeline, but picks their own resolution / audio language.
//
public IReadOnlyList<int> AvailableQualities => MediaStreamOptions.ParseQualities( Qualities );
public IReadOnlyList<MediaAudioTrack> AvailableAudioTracks => MediaStreamOptions.ParseAudioTracks( AudioTracks );
/// <summary> The height this client streams, 0 for automatic. From the bimp_quality convar. </summary>
public int SelectedQuality => MediaStreamOptions.ChooseQuality( AvailableQualities, MediaSettings.Quality );
/// <summary> The audio track this client streams, null for the original. From the bimp_audio_lang convar. </summary>
public MediaAudioTrack SelectedAudioTrack => MediaStreamOptions.ChooseAudioTrack( AvailableAudioTracks, MediaSettings.AudioLanguage );
/// <summary> <see cref="PlayUrl"/> plus this client's choices. Changing a choice reloads playback at the same time. </summary>
public string LocalStreamUrl => MediaStreamOptions.ApplyTo( PlayUrl, SelectedQuality, SelectedAudioTrack );
string CurrentKey => KeyFor( PlayId, LocalStreamUrl );
//
// Captions. Every client fetches its own cues (the urls can be tied to the IP) and shows them against the shared timeline.
//
public IReadOnlyList<MediaCaptionTrack> AvailableCaptionTracks => Captions.ParseTracks( CaptionTracks );
/// <summary> The caption track this client shows, null when captions are off or there are none. From the bimp_subs / bimp_sub_lang convars. </summary>
public MediaCaptionTrack SelectedCaptionTrack => MediaSettings.Subtitles ? Captions.Choose( AvailableCaptionTracks, MediaSettings.SubtitleLanguage ) : null;
/// <summary> The caption line to show right now, null for none. </summary>
public string CurrentCaption => HasMedia ? Captions.TextAt( captionCues, CurrentTime ) : null;
string captionKey;
int captionToken;
List<CaptionCue> captionCues;
/// <summary> Fetch the cues when the media or the chosen track changes. </summary>
void UpdateCaptions()
{
var track = HasMedia ? SelectedCaptionTrack : null;
var key = track is null ? null : $"{PlayId}|{track.Id}";
if ( key == captionKey ) return;
captionKey = key;
captionCues = null;
var token = ++captionToken;
if ( track is not null ) LoadCaptionsAsync( token, track ).FireAndForget( "captions" );
}
async Task LoadCaptionsAsync( int token, MediaCaptionTrack track )
{
try
{
var play = Bimp.Resolver.PlayToken.Parse( PlayUrl );
if ( Bimp.Resolver.Extractors.ExtractorRegistry.ByKey( play?.Extractor ) is not Bimp.Resolver.ICaptionExtractor source ) return;
var body = await source.GetCaptionsAsync( play.Id, track.Id, System.Threading.CancellationToken.None );
if ( token != captionToken || !this.IsValid() ) return;
var cues = Captions.Parse( body );
Log.Info( $"[bimp] captions {track.Id} ({track.Name}): {cues.Count} cues" );
captionCues = cues;
}
catch ( Exception e )
{
Log.Warning( $"[bimp] Couldn't load captions: {e.Message}" );
}
}
//
// Local state
//
/// <summary>
/// The local playback. Null on a dedicated server or when nothing is playing.
/// </summary>
public MediaBackend Backend { get; private set; }
string backendKey;
RealTimeSince sinceCorrection;
bool initialSeekPending;
int reportedDurationFor = -1;
int endedPlayId = -1;
int resolveToken;
/// <summary>
/// Local message for this client (permission denied, load failures), fades after a few seconds.
/// </summary>
public string LocalNotice { get; private set; }
RealTimeSince sinceNotice;
public bool HasMedia => !string.IsNullOrEmpty( PlayUrl );
/// <summary>
/// Are we the authority on the state? True for the host, or when not networked at all.
/// </summary>
public bool IsAuthority => !Networking.IsActive || Networking.IsHost;
/// <summary>
/// Where playback should be right now, according to the host.
/// </summary>
public float CurrentTime
{
get
{
if ( !HasMedia ) return 0;
var t = Paused ? PausedAt : (float)Math.Max( 0, Time.NowDouble - StartTime );
if ( MediaDuration > 0 ) t = MathF.Min( t, MediaDuration );
return t;
}
}
/// <summary>
/// The shared resolver (host only).
/// </summary>
static BimpResolver Resolver => BimpResolver.Instance;
public Vector3 SoundPosition => SoundOrigin.IsValid() ? SoundOrigin.WorldPosition : WorldPosition;
BoxCollider soundBox;
/// <summary> The object the sound comes from: <see cref="SoundOrigin"/>, or this one. </summary>
GameObject SoundObject => SoundOrigin.IsValid() ? SoundOrigin : GameObject;
/// <summary>
/// How much bigger (or smaller) than its prefab this player has been scaled, e.g. by the Sandbox resize tool.
/// The audio range grows with it, so a bigger screen is heard from further away.
/// </summary>
public float SoundScale
{
get
{
var s = SoundObject.WorldScale;
return MathF.Max( 0.01f, MathF.Max( MathF.Abs( s.x ), MathF.Max( MathF.Abs( s.y ), MathF.Abs( s.z ) ) ) );
}
}
/// <summary> <see cref="AudioDistance"/> at the current size. </summary>
public float ScaledAudioDistance => AudioDistance * SoundScale;
/// <summary>
/// Where the sound is heard from for a listener. A scaled-up screen is a big surface, not a point, so the sound comes
/// from the nearest point of its box (the listener's own position when inside it), and fades from there.
/// </summary>
Vector3 SoundPositionFor( Vector3 listener )
{
if ( SoundOrigin.IsValid() ) return SoundOrigin.WorldPosition;
if ( !soundBox.IsValid() ) soundBox = GetComponent<BoxCollider>();
if ( !soundBox.IsValid() ) return WorldPosition;
var tx = soundBox.GameObject.WorldTransform;
var half = soundBox.Scale * 0.5f;
var local = tx.PointToLocal( listener ) - soundBox.Center;
local = new Vector3( local.x.Clamp( -half.x, half.x ), local.y.Clamp( -half.y, half.y ), local.z.Clamp( -half.z, half.z ) );
return tx.PointToWorld( local + soundBox.Center );
}
protected override void OnAwake()
{
// Sync'd properties only work on network objects. Make that the default so mappers don't have to.
if ( GameObject.NetworkMode == NetworkMode.Snapshot )
GameObject.NetworkMode = NetworkMode.Object;
}
protected override void OnStart()
{
if ( IsAuthority && !HasMedia && !string.IsNullOrWhiteSpace( DefaultUrl ) )
{
PlayNowAsync( DefaultUrl, "Map" ).FireAndForget( "map autoplay" );
}
}
protected override void OnDestroy()
{
DisposeBackend();
}
protected override void OnDisabled()
{
DisposeBackend();
}
protected override void OnUpdate()
{
if ( IsAuthority )
UpdateAuthority();
if ( !Application.IsDedicatedServer )
{
UpdatePlayback();
UpdateCaptions();
RecentLinks.Saw( this );
}
MediaProbe.PresentFile();
LatencyLab.Tick();
MediaProbe.SampleLatency( this );
if ( MediaProbe.IsRecording( this ) )
MediaProbe.Sample( this );
if ( LocalNotice is not null && sinceNotice > 6 )
LocalNotice = null;
}
#region Permissions
/// <summary>
/// Can this connection control playback?
/// </summary>
public bool CanControl( Connection c )
{
if ( !Networking.IsActive ) return true;
if ( c is null ) return false;
if ( c.IsHost || c == Connection.Host ) return true;
return Permission switch
{
MediaPermission.Anyone => true,
MediaPermission.Whitelist => AllowedSteamIds?.Contains( c.SteamId.Value ) ?? false,
_ => false,
};
}
/// <summary>
/// Can this connection add to the queue?
/// </summary>
public bool CanQueue( Connection c ) => AnyoneCanQueue || CanControl( c );
/// <summary>
/// Can the local player control playback? Used by the UI.
/// </summary>
public bool CanLocalControl => CanControl( Connection.Local );
public bool CanLocalQueue => CanQueue( Connection.Local );
static string CallerName( Connection c ) => c?.DisplayName ?? "Someone";
#endregion
#region Requests (called by anyone, run on the host)
/// <summary>
/// Play this url now, replacing whatever is playing.
/// </summary>
[Rpc.Host]
public void RequestPlay( string url )
{
var caller = Rpc.Caller;
if ( !CanControl( caller ) ) { Deny( caller, "You don't have permission to control this player." ); return; }
PlayNowAsync( url, CallerName( caller ) ).FireAndForget( $"play from {CallerName( caller )}" );
}
/// <summary>
/// Add this url to the queue. Plays immediately if nothing is playing.
/// </summary>
[Rpc.Host]
public void RequestEnqueue( string url )
{
var caller = Rpc.Caller;
if ( !CanQueue( caller ) ) { Deny( caller, "You don't have permission to queue on this player." ); return; }
if ( Queue.Count >= MaxQueue ) { Deny( caller, "The queue is full." ); return; }
EnqueueAsync( url, CallerName( caller ), caller ).FireAndForget( $"enqueue from {CallerName( caller )}" );
}
[Rpc.Host]
public void RequestSetPaused( bool paused )
{
if ( !CanControl( Rpc.Caller ) ) { Deny( Rpc.Caller, "You don't have permission to control this player." ); return; }
SetPaused( paused );
}
[Rpc.Host]
public void RequestSeek( float time )
{
if ( !CanControl( Rpc.Caller ) ) { Deny( Rpc.Caller, "You don't have permission to control this player." ); return; }
Seek( time );
}
[Rpc.Host]
public void RequestSkip()
{
if ( !CanControl( Rpc.Caller ) ) { Deny( Rpc.Caller, "You don't have permission to control this player." ); return; }
PlayNext();
}
[Rpc.Host]
public void RequestStop()
{
if ( !CanControl( Rpc.Caller ) ) { Deny( Rpc.Caller, "You don't have permission to control this player." ); return; }
Stop();
}
[Rpc.Host]
public void RequestRemove( int index )
{
if ( !CanControl( Rpc.Caller ) ) { Deny( Rpc.Caller, "You don't have permission to control this player." ); return; }
if ( index < 0 || index >= Queue.Count ) return;
Queue.RemoveAt( index );
}
[Rpc.Host]
public void RequestClearQueue()
{
if ( !CanControl( Rpc.Caller ) ) { Deny( Rpc.Caller, "You don't have permission to control this player." ); return; }
Queue.Clear();
}
/// <summary>
/// A client learned the duration of a direct file after loading it. The host takes the first answer.
/// </summary>
[Rpc.Host( NetFlags.Unreliable )]
void ReportDuration( int playId, float duration )
{
if ( playId != PlayId || MediaDuration > 0 || IsLive ) return;
if ( duration <= 0 || float.IsNaN( duration ) || float.IsInfinity( duration ) ) return;
MediaDuration = duration;
}
/// <summary>
/// Tell one client something went wrong with their request.
/// </summary>
void Deny( Connection caller, string message )
{
if ( caller is null || caller == Connection.Local || !Networking.IsActive )
{
ShowNotice( message );
return;
}
using ( Rpc.FilterInclude( caller ) )
{
ReceiveNotice( message );
}
}
[Rpc.Broadcast( NetFlags.HostOnly )]
void ReceiveNotice( string message )
{
ShowNotice( message );
}
void ShowNotice( string message )
{
LocalNotice = message;
sinceNotice = 0;
}
#endregion
#region Authority (host) logic
/// <summary>
/// Resolve and play right away. Host only.
/// </summary>
public async Task PlayNowAsync( string url, string requestedBy )
{
if ( !IsAuthority ) return;
var token = ++resolveToken;
Status = "Loading...";
try
{
// a playlist link: the first video plays now, the rest are queued
var playlist = await Resolver.ResolvePlaylistAsync( url, AudioOnlyPlayer, MaxQueue + 1 );
if ( token != resolveToken || !this.IsValid() ) return;
if ( playlist is not null )
{
var first = await ResolveLazyAsync( playlist[0], requestedBy );
if ( token != resolveToken || !this.IsValid() ) return;
StartItem( first );
QueueLazy( playlist.Skip( 1 ), requestedBy, null );
return;
}
var item = await Resolver.ResolveAsync( url, AudioOnlyPlayer );
if ( token != resolveToken || !this.IsValid() ) return; // something newer was requested
item.RequestedBy = requestedBy;
StartItem( item );
}
catch ( Exception e )
{
if ( token != resolveToken || !this.IsValid() ) return;
Status = e is ResolveException ? e.Message : $"Error: {e.Message}";
Log.Warning( $"[bimp] Couldn't play {url}: {(e is ResolveException ? e.Message : e.ToString())}" );
}
}
/// <summary>
/// Resolve and add to the queue. Host only.
/// </summary>
public async Task EnqueueAsync( string url, string requestedBy, Connection caller = null )
{
if ( !IsAuthority ) return;
try
{
var playlist = await Resolver.ResolvePlaylistAsync( url, AudioOnlyPlayer, MaxQueue + 1 );
if ( !this.IsValid() ) return;
if ( playlist is not null )
{
var rest = playlist.AsEnumerable();
if ( !HasMedia && Status != "Loading..." )
{
var first = await ResolveLazyAsync( playlist[0], requestedBy );
if ( !this.IsValid() ) return;
if ( !HasMedia && Status != "Loading..." ) // nothing else started meanwhile
{
StartItem( first );
rest = playlist.Skip( 1 );
}
}
QueueLazy( rest, requestedBy, caller );
return;
}
var item = await Resolver.ResolveAsync( url, AudioOnlyPlayer );
if ( !this.IsValid() ) return;
item.RequestedBy = requestedBy;
if ( !HasMedia && Status != "Loading..." )
{
StartItem( item );
return;
}
if ( Queue.Count >= MaxQueue ) { Deny( caller, "The queue is full." ); return; }
Queue.Add( item );
}
catch ( Exception e )
{
if ( !this.IsValid() ) return;
Deny( caller, e is ResolveException ? e.Message : $"Error: {e.Message}" );
}
}
/// <summary> A playlist entry resolved properly (qualities, dubs, live or not). Throws like <see cref="BimpResolver.ResolveAsync"/>. </summary>
async Task<MediaQueueItem> ResolveLazyAsync( MediaQueueItem entry, string requestedBy )
{
var item = await Resolver.ResolveAsync( entry.Url, AudioOnlyPlayer );
item.RequestedBy = requestedBy;
return item;
}
/// <summary> Add playlist entries to the queue until it's full, and say so if some didn't fit. </summary>
void QueueLazy( IEnumerable<MediaQueueItem> entries, string requestedBy, Connection caller )
{
int added = 0, left = 0;
foreach ( var entry in entries )
{
if ( Queue.Count >= MaxQueue ) { left++; continue; }
entry.RequestedBy = requestedBy;
Queue.Add( entry );
added++;
}
if ( left > 0 ) Deny( caller, $"Queued {added} videos - the queue is full, {left} more didn't fit." );
}
/// <summary> Play a queued playlist entry: resolve it first, and skip it (with a notice) if it can't be played. </summary>
async Task StartLazyAsync( MediaQueueItem entry )
{
var token = resolveToken;
Status = "Loading...";
try
{
var item = await ResolveLazyAsync( entry, entry.RequestedBy );
if ( token != resolveToken || !this.IsValid() ) return;
StartItem( item );
}
catch ( Exception e )
{
if ( token != resolveToken || !this.IsValid() ) return;
var why = e is ResolveException ? e.Message : $"Error: {e.Message}";
Log.Warning( $"[bimp] Couldn't play {entry.Url}: {e.Message}" );
var notice = $"Skipped \"{entry.Title}\": {why}";
if ( Networking.IsActive ) ReceiveNotice( notice );
else ShowNotice( notice );
PlayNext();
}
}
void StartItem( MediaQueueItem item )
{
CurrentUrl = item.Url;
PlayUrl = item.PlayUrl;
Title = item.Title;
RequestedBy = item.RequestedBy;
MediaDuration = item.Duration;
IsLive = item.IsLive;
AudioOnly = item.AudioOnly || AudioOnlyPlayer;
SeekByReload = item.SeekByReload;
Qualities = item.Qualities;
AudioTracks = item.AudioTracks;
CaptionTracks = item.CaptionTracks;
Paused = false;
PausedAt = 0;
// a link with a start time (?t=90) plays from there, the way a late joiner starts mid-way
StartTime = Time.NowDouble - (item.IsLive ? 0 : Math.Max( 0, item.StartAt ));
Status = null;
PlayId++;
}
/// <summary>
/// Play the next item in the queue, or stop. Host only.
/// </summary>
public void PlayNext()
{
if ( !IsAuthority ) return;
resolveToken++; // cancel any pending "play now"
if ( Queue.Count > 0 )
{
var next = Queue[0];
Queue.RemoveAt( 0 );
if ( next.Lazy ) StartLazyAsync( next ).FireAndForget( $"lazy start of {next.Title}" );
else StartItem( next );
return;
}
Stop();
}
/// <summary>
/// Stop playback (the queue is kept). Host only.
/// </summary>
public void Stop()
{
if ( !IsAuthority ) return;
resolveToken++;
CurrentUrl = null;
PlayUrl = null;
Title = null;
RequestedBy = null;
Qualities = null;
AudioTracks = null;
CaptionTracks = null;
MediaDuration = 0;
IsLive = false;
Paused = false;
PausedAt = 0;
Status = null;
PlayId++;
}
public void SetPaused( bool paused )
{
if ( !IsAuthority || !HasMedia || IsLive || Paused == paused ) return;
if ( paused )
{
// Freeze the shared timeline where the host's picture actually is, so the host's own video
// doesn't jump on pause. Clients are within their drift tolerance of it anyway.
PausedAt = LocalPlaybackPosition() ?? CurrentTime;
Paused = true;
}
else
{
StartTime = Time.NowDouble - PausedAt;
Paused = false;
}
}
public void Seek( float time )
{
if ( !IsAuthority || !HasMedia || IsLive ) return;
time = MathF.Max( 0, time );
if ( MediaDuration > 0 ) time = MathF.Min( time, MediaDuration - 0.5f );
if ( Paused ) PausedAt = time;
else StartTime = Time.NowDouble - time;
SeekId++;
// Don't let an old "ended" state trip us up after seeking back
endedPlayId = -1;
}
void UpdateAuthority()
{
if ( !HasMedia || Paused || IsLive ) return;
if ( endedPlayId == PlayId ) return;
var ended = false;
// Known duration - use the clock, works on dedicated servers too
if ( MediaDuration > 0 && Time.NowDouble - StartTime >= MediaDuration + 1.0 )
ended = true;
// Unknown duration - trust our own playback, if we have one
if ( MediaDuration <= 0 && Backend is not null && Backend.Finished && backendKey == CurrentKey )
ended = true;
if ( !ended ) return;
endedPlayId = PlayId;
OnMediaEnded();
}
void OnMediaEnded()
{
if ( Queue.Count > 0 )
{
PlayNext();
return;
}
if ( Loop )
{
Paused = false;
StartTime = Time.NowDouble;
PlayId++; // force everyone to reload, the decoder has stopped
return;
}
Stop();
}
#endregion
#region Local playback
static string KeyFor( int playId, string url ) => string.IsNullOrEmpty( url ) ? null : $"{playId}|{url}";
/// <summary>
/// Throw away local playback and start it again at the synced time.
/// </summary>
internal void ReloadBackend()
{
DisposeBackend();
backendKey = CurrentKey;
if ( backendKey is not null ) CreateBackend();
}
/// <summary>
/// Diagnostics: open merged media at t with an explicit time offset.
/// </summary>
internal void ReloadBackendAt( float t, float offset )
{
DisposeBackend();
backendKey = CurrentKey;
if ( backendKey is null ) return;
Backend = PlayToken.IsToken( LocalStreamUrl )
? MediaBackend.CreateNative( LocalStreamUrl, AudioOnly, t )
: MediaBackend.Create( LocalStreamUrl, AudioOnly, offset );
sinceCorrection = 0;
initialSeekPending = false;
appliedSeekId = SeekId;
lastAppliedPaused = !Paused;
}
void DisposeBackend()
{
Backend?.Dispose();
Backend = null;
backendKey = null;
}
void CreateBackend()
{
// Merged media starts at the synced time: segment files are written from the cue point before it,
// with absolute timestamps, so there's no time offset. Everything else starts at 0 and seeks once loaded.
var t = CurrentTime;
var url = LocalStreamUrl;
Backend = IsLive ? MediaBackend.CreateLive( url, Math.Clamp( MediaSettings.MaxVideoHeight, 144, 2160 ), AudioOnly )
: PlayToken.IsToken( url ) ? MediaBackend.CreateNative( url, AudioOnly, SeekByReload && !IsLive ? t : 0 )
: MediaBackend.Create( url, AudioOnly, 0 );
sinceCorrection = 0;
initialSeekPending = !IsLive; // once loaded, land exactly on the synced time (see CorrectDrift)
appliedSeekId = SeekId; // a fresh backend starts at the right time anyway (initial seek)
lastAppliedPaused = !Paused; // make sure the new backend gets told the pause state
if ( Backend.Error is not null )
{
notifiedError = Backend;
ShowNotice( $"Can't play this: {Backend.Error}" );
}
}
void UpdatePlayback()
{
var wantKey = CurrentKey;
if ( wantKey != backendKey )
{
DisposeBackend();
backendKey = wantKey;
if ( wantKey is not null )
CreateBackend();
}
if ( Backend is null ) return;
Backend.SyncedTime = Paused || IsLive ? null : CurrentTime;
Backend.Present();
var volume = Volume * MediaSettings.EffectiveVolume;
if ( FlatWithDistance ) volume *= DistanceFade( Sound.Listener.Position );
Backend.SetAudio( SoundPositionFor( Sound.Listener.Position ), EffectiveSpatial, volume, ScaledAudioDistance );
if ( Backend.Error is not null )
{
// the stream's urls were refused (403/410) even after a refresh - re-resolve from the next client,
// the same way a seek reopens, instead of sitting on the error
if ( Backend.Refused && TryRefusalReload() )
return;
// native media fails after the backend was created (resolving / downloading) - say so once
if ( notifiedError != Backend )
{
notifiedError = Backend;
ShowNotice( $"Can't play this: {Backend.Error}" );
}
return;
}
if ( !Backend.Loaded )
{
if ( Backend.SinceCreated > 20 && LocalNotice is null )
ShowNotice( "This is taking a while to load..." );
return;
}
// Tell the host the duration of direct files, it can't know until something loads them
if ( MediaDuration <= 0 && !IsLive && !SeekByReload && reportedDurationFor != PlayId && Backend.Duration > 0 )
{
reportedDurationFor = PlayId;
ReportDuration( PlayId, Backend.Duration );
}
// Pausing/resuming gets a moment to settle (the decoder refills its buffers on resume) before
// anything is judged as drift or a stall - correcting then is what caused stutter after unpausing
// A freshly opened player (after a seek, join or reload) runs until it has landed on the synced time,
// even when paused - otherwise a seek while paused leaves the screen without the new frame
var wantPaused = Paused && !initialSeekPending;
if ( wantPaused != lastAppliedPaused )
{
lastAppliedPaused = wantPaused;
sinceCorrection = 0;
ResetStallTimer();
}
Backend.SetPaused( wantPaused );
// A stuck live stream (source hiccup, dropped connection) reconnects at the live edge
if ( CheckStalled() ) return;
if ( IsLive ) return; // nothing to sync to - everyone watches the live edge
CorrectDrift();
}
bool lastAppliedPaused;
int appliedSeekId;
MediaBackend notifiedError;
/// <summary>
/// Where this client's playback actually is, if it's loaded and roughly in sync (null otherwise).
/// </summary>
float? LocalPlaybackPosition()
{
if ( Backend is null || Backend.Error is not null || !Backend.Loaded || backendKey != CurrentKey ) return null;
var t = Backend.Time;
if ( MathF.Abs( t - CurrentTime ) > 3.0f ) return null; // way off (stalled / still seeking) - don't trust it
if ( MediaDuration > 0 ) t = MathF.Min( t, MediaDuration );
return MathF.Max( 0, t );
}
const float StallSeconds = 5.0f;
const int MaxStallReloads = 3;
float lastProgressTime;
RealTimeSince sinceProgress;
int stallReloads;
int stallPlayId = -1;
/// <summary>
/// If playback should be moving but hasn't for a while (the decoder got stuck after a seek,
/// the connection dropped...), throw the backend away and start again at the right time.
/// Returns true if it reloaded.
/// </summary>
bool CheckStalled()
{
if ( stallPlayId != PlayId )
{
stallPlayId = PlayId;
stallReloads = 0;
ResetStallTimer();
}
var t = Backend.Time;
if ( Paused || MathF.Abs( t - lastProgressTime ) > 0.05f )
{
lastProgressTime = t;
sinceProgress = 0;
return false;
}
// Reached the end, not stuck
if ( MediaDuration > 0 && t >= MediaDuration - 1.0f ) return false;
if ( sinceProgress < StallSeconds || stallReloads >= MaxStallReloads ) return false;
stallReloads++;
Log.Warning( $"[bimp] Playback stalled at {t:0.0}s, reloading ({stallReloads}/{MaxStallReloads})" );
ShowNotice( "Playback got stuck, reloading..." );
DisposeBackend();
backendKey = CurrentKey;
CreateBackend();
ResetStallTimer();
return true;
}
void ResetStallTimer()
{
lastProgressTime = -1;
sinceProgress = 0;
}
const int MaxRefusalReloads = 2;
int refusalReloads;
int refusalPlayId = -1;
/// <summary>
/// The stream's urls were refused even after a refresh: forget the cached session and reload, so it resolves
/// fresh from the next source (StreamSessions skips the client that just died). Budgeted like a stall.
/// </summary>
bool TryRefusalReload()
{
if ( refusalPlayId != PlayId ) { refusalPlayId = PlayId; refusalReloads = 0; }
if ( refusalReloads >= MaxRefusalReloads ) return false;
refusalReloads++;
Log.Warning( $"[bimp] stream refused at {Backend.Time:0.0}s, re-resolving from another source ({refusalReloads}/{MaxRefusalReloads})" );
ShowNotice( "The stream stopped answering, switching source..." );
Backend.InvalidateSession();
ReloadBackend();
ResetStallTimer();
return true;
}
/// <summary> Merged media lands within this of the synced time without seeking (the drift threshold). </summary>
const float MergedLandingTolerance = 1.5f;
/// <summary> The play and seek that merged media last reopened for, after landing too far behind. </summary>
(int playId, int seekId) landingReloadFor = (-1, -1);
/// <summary>
/// Keep local playback close to where the host says it should be.
/// </summary>
void CorrectDrift()
{
var expected = CurrentTime;
var drift = Backend.Time - expected;
var absDrift = MathF.Abs( drift );
// Just loaded: land exactly on the synced time. For merged media the first segment starts at the keyframe
// cluster before it, so this is a short hop forward inside the first cluster (a few seconds at most),
// which the engine's player gets through in a blink.
if ( initialSeekPending )
{
// The loaded event can come a frame or two before the player's clock reports the segment's position
// (it reads 0) - wait for it, or we'd skip the hop and sit a second behind
if ( SeekByReload && absDrift >= 15.0f && Backend.SinceCreated < 10.0f ) return;
initialSeekPending = false;
appliedSeekId = SeekId;
// Merged media started at a keyframe just behind the synced time (or waited for the next one): it keeps that
// small lag rather than seeking forward, which means decoding everything in between while the sound runs
// on - at 4K the picture then crawled behind for seconds (see MediaBackend.StartNative).
MediaProbe.Note( $"initial landing: drift {drift * 1000:0}ms" );
var tolerance = SeekByReload ? MergedLandingTolerance : 0.25f;
if ( absDrift > tolerance && (!SeekByReload || absDrift < 15.0f) )
{
// Landed further behind than planned (a slow load): open again, now planned with the real load time,
// rather than seek forward. Once per play or seek, so a load that's always slow can't loop.
if ( SeekByReload && drift < 0 && landingReloadFor != (PlayId, SeekId) )
{
landingReloadFor = (PlayId, SeekId);
MediaProbe.Note( "landed too far behind: reopening" );
ReloadBackend();
return;
}
Backend.Seek( expected );
sinceCorrection = 0;
}
return;
}
// Someone seeked (button / progress bar): apply it now, not as drift after a cooldown
var explicitSeek = appliedSeekId != SeekId;
appliedSeekId = SeekId;
// The same tolerance applies while paused - pausing must never seek (our playback normally trails
// the synced clock by up to a second of load latency; correcting that on pause made it jump).
const float threshold = 1.5f;
var drifted = absDrift >= threshold && sinceCorrection >= 3.0f;
if ( !(explicitSeek && absDrift > 0.25f) && !drifted ) return;
if ( SeekByReload )
{
// ROOT CAUSE of "it keeps skipping forward frames": the engine's player can't seek forward in a
// stream. Seek() jumps its clock but it keeps reading and decoding sequentially from where it was,
// throwing away every frame as late until it catches up - fast forwarding for as long as the gap
// was (minutes, for a big seek), while drift correction kept firing more seeks. So merged
// media never asks it to jump: it reopens at the target, writing segment files that start there.
if ( MediaDuration > 0 && expected >= MediaDuration - 2 ) return;
ReloadBackend();
return;
}
// Direct files: the native seek (fine for backward seeks and short hops)
Backend.Seek( expected );
sinceCorrection = 0;
}
#endregion
}
Game
library
using System;
using Bimp.Resolver;
using Bimp.Resolver.Media;
namespace Bimp;
/// <summary>
/// Local playback of a single url or play token. Wraps the engine's <see cref="VideoPlayer"/> (video),
/// <see cref="MusicPlayer"/> (audio only, gives us a spectrum) or a <see cref="SegmentPlayer"/> (merged
/// YouTube media) behind one interface.
/// Each client owns one of these per <see cref="MediaPlayer"/>, it is never networked.
/// </summary>
public sealed class MediaBackend : IDisposable
{
VideoPlayer video;
MusicPlayer music;
/// <summary> The direct video player's frames (see <see cref="VideoFrameSink"/>). </summary>
readonly VideoFrameSink videoSink = new();
SegmentPlayer segments;
WebmSegmenter segmenter;
Resolver.Live.LiveStream live;
/// <summary>
/// Natively resolved media being prepared (resolving, downloading the first segment).
/// </summary>
Task<StreamSession> preparing;
string token;
string sessionClient;
bool audioOnly;
float startTime;
bool disposed;
/// <summary>
/// Merged media: how long from starting the segmenter to its first segment being ready (download + write),
/// averaged over every backend. A seek or join plans its first segment from where the synced time will be by then.
/// </summary>
static float firstSegmentSeconds = 1.5f;
RealTimeSince sinceSegmenter;
bool firstSegmentTimed;
/// <summary> Checking / fixing an MP4 before playing it (see <see cref="Mp4ChannelFix"/>). </summary>
bool checkingMp4;
string localDirectory;
readonly System.Threading.CancellationTokenSource cts = new();
/// <summary>
/// For streams that were started at an offset and whose clock starts at 0 - this is the offset.
/// </summary>
public float TimeOffset { get; private set; }
public string Url { get; private set; }
public bool IsAudioOnly => music is not null;
/// <summary>
/// Still resolving / downloading before anything can play.
/// </summary>
public bool IsPreparing => preparing is not null || checkingMp4;
/// <summary>
/// Something went wrong, user facing message.
/// </summary>
string error;
public string Error { get => error ?? live?.Error ?? segments?.Error; private set => error = value; }
/// <summary> A segment build was refused (403/410) even after the url was refreshed - only a fresh session, from another client, helps. </summary>
public bool Refused => segmenter?.Refused ?? false;
/// <summary>
/// Forget this media's cached session, so the next backend resolves fresh - skipping the client whose urls
/// just refused (see MediaPlayer's refusal reload).
/// </summary>
public void InvalidateSession()
{
if ( token is not null ) StreamSessions.Forget( token, sessionClient );
}
/// <summary> Live: what we're connected to ("RTSP H.264 1920x1080 + G.711 mu-law"). </summary>
public string LiveDescription => live is null ? null
: $"{live.Description} | behind newest frame {LiveBehindNewest:0.00}s{(LiveLatency is { } l ? $", sent {l:0.00}s ago" : ", no sender clock")}{(live.Segmenter.Pcm is { } pcm ? $", audio {pcm.SampleRate}Hz {pcm.Channels}ch, A/V {pcm.Offset * 1000:+0;-0}ms ({pcm.State})" : "")}";
/// <summary> Live, decoded audio: its stream time minus the picture's (seconds, negative = audio late). </summary>
public double? LiveAvOffset => live?.Segmenter.Pcm?.Offset;
/// <summary>
/// Live: how long ago the frame on screen was sent, by the sender's clock (RTCP sender reports) - the latency
/// from the source's output to our screen. Null without sender reports (MPEG-TS, HLS, servers that send none).
/// </summary>
public double? LiveLatency => live?.Segmenter.WallClockZero is { } zero && segments is { Loaded: true }
? (DateTime.UtcNow - DateTime.UnixEpoch).TotalSeconds - (zero + segments.StreamTime)
: null;
/// <summary> Live: stream time from the frame on screen to the newest frame received. </summary>
public double LiveBehindNewest => live is null || segments is not { Loaded: true } ? 0 : live.Segmenter.NewestVideoTime - segments.StreamTime;
/// <summary> Merged media opened before this media time starts from the beginning (see <see cref="StartNative"/>). </summary>
const float FreshStartSeconds = 1.0f;
/// <summary>
/// Merged media: start at the keyframe before the planned time if it's at most this far behind it, else at the next
/// one (waiting for it). Stays inside the drift tolerance (<see cref="MediaPlayer"/>, 1.5 s) with the estimate's error.
/// </summary>
const float MaxStartLag = 1.0f;
/// <summary>
/// Where the synced timeline is, while it's moving (null while paused). Merged media that starts at a later keyframe
/// waits for it before opening its first segment.
/// </summary>
public float? SyncedTime { set { if ( segments is not null ) segments.SyncedTime = value; } }
/// <summary>
/// Playback has reached the end.
/// </summary>
bool finished;
public bool Finished { get => finished || (segments?.Finished ?? false); private set => finished = value; }
bool loaded;
float lastPlaybackTime;
/// <summary>
/// Has the media actually started (metadata loaded / audio is advancing)?
/// </summary>
public bool Loaded => segments?.Loaded ?? (!IsPreparing && (loaded || PlaybackTime > 0.01f));
/// <summary>
/// Time since this backend was created.
/// </summary>
public RealTimeSince SinceCreated { get; private set; }
/// <summary>
/// Position within the media, taking the start offset into account.
/// </summary>
public float Time => TimeOffset + PlaybackTime;
public float PlaybackTime => segments?.Time ?? video?.PlaybackTime ?? music?.PlaybackTime ?? 0;
/// <summary>
/// Duration as reported by the decoder.
/// </summary>
public float Duration => segments?.Duration ?? video?.Duration ?? music?.Duration ?? 0;
public bool IsPaused => segments?.IsPaused ?? video?.IsPaused ?? music?.Paused ?? false;
public Texture Texture => segments?.Texture ?? videoSink.Texture;
public int Width => segments?.Width ?? (video is null ? 0 : videoSink.Width);
public int Height => segments?.Height ?? (video is null ? 0 : videoSink.Height);
/// <summary>
/// Does the video have visible frames?
/// </summary>
public bool HasVideo => segments?.HasVideo ?? (video is not null && videoSink.Texture is not null && videoSink.Width > 4 && videoSink.Height > 4);
/// <summary>
/// 512 FFT values for visualizers. Empty for video playback.
/// </summary>
public ReadOnlySpan<float> Spectrum => music is not null ? music.Spectrum : ReadOnlySpan<float>.Empty;
public float Amplitude => music?.Amplitude ?? 0;
/// <summary>
/// The decoded audio format, e.g. "48000Hz 2ch" (for diagnostics). Empty until audio has started.
/// </summary>
public string AudioFormat
{
get
{
var (rate, ch) = segments is not null ? segments.AudioFormat : video is not null ? (video.SampleRate, video.Channels) : music is not null ? (music.SampleRate, music.Channels) : (0, 0);
return rate > 0 ? $"{rate}Hz {ch}ch" : "";
}
}
MediaBackend() { }
/// <summary>
/// Start playing <paramref name="url"/>. Never throws, check <see cref="Error"/>.
/// </summary>
public static MediaBackend Create( string url, bool audioOnly, float timeOffset )
{
var b = new MediaBackend
{
Url = url,
TimeOffset = timeOffset,
SinceCreated = 0,
};
b.StartUrl( url, audioOnly );
return b;
}
/// <summary>
/// Resolve a play token on this client and start playing it at <paramref name="startTime"/> (media seconds).
/// Never throws, check <see cref="Error"/>.
/// </summary>
public static MediaBackend CreateNative( string token, bool audioOnly, float startTime )
{
var b = new MediaBackend
{
Url = token,
SinceCreated = 0,
token = token,
audioOnly = audioOnly,
startTime = startTime,
};
b.preparing = StreamSessions.GetAsync( token );
return b;
}
/// <summary>
/// Connect to a live stream (RTSP tunnelled over HTTP, MPEG-TS over HTTP, HLS) and play it near its live edge.
/// Never throws, check <see cref="Error"/>.
/// </summary>
public static MediaBackend CreateLive( string url, int maxHeight, bool audioOnly = false )
{
var b = new MediaBackend { Url = url, SinceCreated = 0, audioOnly = audioOnly, liveMaxHeight = audioOnly ? AudioOnlyHeight : maxHeight };
try
{
// a live channel from an extractor (Twitch...): resolved on this client first (see StartNative)
if ( PlayToken.IsToken( url ) )
{
b.token = url;
b.preparing = StreamSessions.GetAsync( url );
return b;
}
b.StartLive( url, false );
}
catch ( Exception e )
{
b.Error = e.Message;
}
return b;
}
int liveMaxHeight = 720;
/// <summary>
/// The variant height an audio-only player (a speaker) asks a live or HLS stream for: the audio comes with every
/// variant, so the smallest one saves the download and the decoding of a picture nobody sees.
/// </summary>
const int AudioOnlyHeight = 144;
void StartLive( string url, bool hls )
{
live = new Resolver.Live.LiveStream( url, StreamSessions.NewCacheDirectory(), liveMaxHeight, hls );
segments = new SegmentPlayer( live.Segmenter );
}
/// <summary> The height a play token asks for (its pick, else the server's limit). </summary>
static int TokenHeight( string token, int fallback )
=> PlayToken.Parse( token ) is { } t ? (t.Height > 0 ? t.Height : t.MaxHeight) : fallback;
void StartUrl( string url, bool audio )
{
// MP4s whose header mislabels mono audio as stereo crackle in the engine - check first
if ( Mp4ChannelFix.Applies( url ) )
{
_ = StartMp4( url, audio );
return;
}
PlayUrlNow( url, audio );
}
async Task StartMp4( string url, bool audio )
{
checkingMp4 = true;
try
{
localDirectory = StreamSessions.NewCacheDirectory();
var path = await Mp4ChannelFix.PrepareAsync( url, localDirectory, cts.Token );
if ( disposed ) return;
if ( path is null ) PlayUrlNow( url, audio );
else PlayFile( path, audio );
}
catch ( OperationCanceledException )
{
}
catch ( Exception e )
{
if ( disposed ) return;
// couldn't fix it - play it as it is rather than not at all
Log.Warning( $"[bimp] Couldn't prepare {url}: {e.Message}" );
PlayUrlNow( url, audio );
}
finally
{
checkingMp4 = false;
}
}
void PlayFile( string path, bool audio )
{
try
{
if ( audio )
{
music = MusicPlayer.Play( FileSystem.Data, path );
music.OnFinished += () => Finished = true;
}
else
{
video = new VideoPlayer();
videoSink.Attach( video, null, null, "video" );
video.OnLoaded += () => loaded = true;
video.OnFinished += () => Finished = true;
video.Play( FileSystem.Data, path );
}
}
catch ( Exception e )
{
Error = e.Message;
Log.Warning( $"[bimp] Failed to play {path}: {e.Message}" );
}
}
void PlayUrlNow( string url, bool audio )
{
try
{
if ( audio )
{
music = MusicPlayer.PlayUrl( url );
music.OnFinished += () => Finished = true;
}
else
{
video = new VideoPlayer();
videoSink.Attach( video, null, null, "video" );
video.OnLoaded += () => loaded = true;
video.OnFinished += () => Finished = true;
video.Play( url );
}
_ = CheckReachable( url );
}
catch ( Exception e )
{
Error = e.Message;
Log.Warning( $"[bimp] Failed to play {url}: {e.Message}" );
}
}
/// <summary>
/// The engine's players give up on a url that answers with an error page (a 403, a 404) without saying so - the
/// screen would sit on "loading" for good. Look at the response's status ourselves. Headers only: a radio stream
/// never ends, and a buffered request would never return.
/// </summary>
async Task CheckReachable( string url )
{
if ( !url.StartsWith( "http", StringComparison.OrdinalIgnoreCase ) ) return;
try
{
using var stream = await Http.RequestStreamAsync( url, cancellationToken: cts.Token );
}
catch ( System.Net.Http.HttpRequestException e ) when ( e.StatusCode is { } status && !loaded && !disposed )
{
Error = $"The server refused the file ({(int)status}).";
Log.Warning( $"[bimp] {url}: {(int)status}" );
}
catch ( Exception )
{
// unreachable, cancelled: the engine's own load decides
}
}
/// <summary>
/// The client side resolve finished - start the right kind of playback.
/// </summary>
void StartNative( StreamSession session )
{
var plan = session.Plan;
sessionClient = plan.ClientKey;
if ( plan.Kind == StreamKind.Direct )
{
// a progressive file - played straight from the site, seeked natively once loaded
StartUrl( plan.DirectUrl, audioOnly );
return;
}
if ( plan.Kind == StreamKind.Live )
{
liveMaxHeight = audioOnly ? AudioOnlyHeight : TokenHeight( token, liveMaxHeight );
StartLive( plan.DirectUrl, true );
return;
}
// The engine can only seek forward by decoding every frame in between, while the audio runs on - a 4K video
// can't decode much faster than it plays, so after a seek the picture crawled behind the sound for seconds. So
// merged media starts at a keyframe close behind where the synced time will be once the first segment is
// ready, or at the next one and waits there (see SegmentPlayer.SyncedTime) - it never seeks forward. The start
// of a video just plays from the beginning, a load time behind.
var planAhead = plan.Kind is StreamKind.Merge or StreamKind.Hls && !audioOnly && startTime >= FreshStartSeconds;
var plannedStart = planAhead ? startTime + (float)SinceCreated + firstSegmentSeconds : startTime;
var maxLag = planAhead ? MaxStartLag : float.MaxValue;
sinceSegmenter = 0;
if ( plan.Kind == StreamKind.Hls )
{
// a finished HLS playlist: remuxed from the segment at (or just after) the planned start, like merged media
live = new Resolver.Live.LiveStream( plan.DirectUrl, StreamSessions.NewCacheDirectory(), audioOnly ? AudioOnlyHeight : TokenHeight( token, 720 ), true, plannedStart, maxLag );
segments = new SegmentPlayer( live.Segmenter );
return;
}
segmenter = new WebmSegmenter( plan.Kind == StreamKind.Merge ? session.Video : null, session.Audio, plannedStart, StreamSessions.NewCacheDirectory(), maxLag, session.SwitchVideo );
if ( plan.Kind == StreamKind.Audio || audioOnly )
{
// one file from the cue point before the start time to the end, through MusicPlayer for the spectrum
_ = StartAudioFile();
return;
}
segments = new SegmentPlayer( segmenter );
}
async Task StartAudioFile()
{
preparing = null;
try
{
var segment = await segmenter.GetAsync( 0 );
if ( disposed || segment is null ) return;
music = MusicPlayer.Play( FileSystem.Data, segment.Path );
music.OnFinished += () => Finished = true;
}
catch ( Exception e )
{
if ( disposed ) return;
Error = e.Message;
StreamSessions.Forget( token );
}
}
/// <summary>
/// Call every frame. Pushes the latest video frame to the texture.
/// </summary>
public void Present()
{
if ( preparing is { IsCompleted: true } task )
{
preparing = null;
if ( task.IsCompletedSuccessfully )
{
try
{
StartNative( task.Result );
}
catch ( Exception e )
{
Error = e.Message;
StreamSessions.Forget( token );
}
}
else
{
var e = task.Exception?.InnerException;
Error = e is ResolveException ? e.Message : $"Couldn't load the stream: {e?.Message}";
Log.Warning( $"[bimp] Failed to resolve {token}: {e}" );
StreamSessions.Forget( token );
}
}
if ( !firstSegmentTimed && ((ISegmentFeed)segmenter ?? (live?.Segmenter is { IsLive: false } vod ? vod : null))?.TryGet( 0 ) is not null )
{
firstSegmentTimed = true;
firstSegmentSeconds = Math.Clamp( firstSegmentSeconds * 0.5f + sinceSegmenter * 0.5f, 0.3f, 10.0f );
}
segments?.Update();
if ( video is not null )
{
var presentStart = System.Diagnostics.Stopwatch.GetTimestamp();
video.Present();
MediaProbe.PresentTook( System.Diagnostics.Stopwatch.GetElapsedTime( presentStart ).TotalMilliseconds, "direct", 0 );
}
if ( live is not null && segments is not null )
{
live.PlayingSeq = segments.CurrentSeq;
// camera G.711 audio plays beside the video, lined up with the picture on screen
live.Segmenter.Pcm?.Update( segments.StreamTime, segments.Loaded && requestedPaused != true );
}
// MusicPlayer has no loaded event, so treat advancing time as loaded
var t = PlaybackTime;
if ( t > lastPlaybackTime + 0.01f ) loaded = true;
lastPlaybackTime = t;
}
bool? requestedPaused;
RealTimeSince sincePauseRequest;
/// <summary>
/// Pause or resume. Only calls into the native player when the requested state changes - not by
/// polling its IsPaused every frame, which can lag a call and made us call Pause/Resume repeatedly
/// (every extra Resume restarts the decoder's clock, which shows up as stutter).
/// If the native player still disagrees a while later (e.g. it hadn't started yet), ask once more.
/// </summary>
public void SetPaused( bool paused )
{
if ( requestedPaused == paused )
{
if ( IsPaused == paused || sincePauseRequest < 1.0f ) return;
}
requestedPaused = paused;
sincePauseRequest = 0;
MediaProbe.Note( paused ? "native Pause()" : "native Resume()" );
segments?.SetPaused( paused );
if ( video is not null )
{
if ( paused ) video.Pause(); else video.Resume();
}
if ( music is not null )
music.Paused = paused;
}
/// <summary>
/// Seek to a position within the media (not accounting for <see cref="TimeOffset"/> - pass media time).
/// </summary>
public void Seek( float mediaTime )
{
var local = MathF.Max( 0, mediaTime - TimeOffset );
MediaProbe.Note( $"native Seek({local:0.000}) from {PlaybackTime:0.000}" );
segments?.Seek( local );
video?.Seek( local );
music?.Seek( local );
Finished = false;
}
public void SetAudio( Vector3 position, bool spatial, float volume, float distance )
{
segments?.SetAudio( position, spatial, volume, distance );
live?.Segmenter.Pcm?.SetAudio( position, spatial, volume, distance );
// Non-spatial audio is played at the listener, slightly in front so it's centered
var pos = spatial ? position : Vector3.Forward * 64.0f;
if ( video is not null )
{
var a = video.Audio;
a.ListenLocal = !spatial;
a.Position = pos;
a.Volume = volume;
a.Distance = distance;
a.Falloff = MediaPlayer.AudioFalloff;
}
if ( music is not null )
{
music.ListenLocal = !spatial;
music.Position = pos;
music.Volume = volume;
music.Distance = distance;
music.Falloff = MediaPlayer.AudioFalloff;
}
}
public void Dispose()
{
disposed = true;
preparing = null;
cts.Cancel();
segments?.Dispose();
segments = null;
video?.Dispose();
video = null;
music?.Dispose();
music = null;
videoSink.Dispose();
// after the players, so the files aren't open anymore
segmenter?.Dispose();
segmenter = null;
live?.Dispose();
live = null;
if ( localDirectory is not null )
{
try
{
if ( FileSystem.Data.DirectoryExists( localDirectory ) ) FileSystem.Data.DeleteDirectory( localDirectory, true );
}
catch ( Exception e )
{
Log.Trace( $"[bimp] couldn't delete {localDirectory}: {e.Message}" );
}
localDirectory = null;
}
}
}
Game
library
namespace Bimp;
/// <summary>
/// Lets players use the media player with the USE key (works with the engine's <see cref="PlayerController"/>,
/// or anything else that calls <see cref="Component.IPressable"/>). Needs a collider on this GameObject or a child.
/// <para>
/// Aiming at one of the screen's on-screen buttons while the controls are showing presses that button.
/// Anywhere else (or while the controls are hidden) opens the full remote.
/// </para>
/// </summary>
[Title( "Media Player Interact" ), Category( "Media" ), Icon( "touch_app" )]
public sealed class MediaInteract : Component, Component.IPressable
{
/// <summary>
/// The player to control. Found automatically in this object or its parents if unset.
/// </summary>
[Property] public MediaPlayer Player { get; set; }
/// <summary>
/// The screen whose on-screen buttons USE can press. Found automatically in this object or its children if unset.
/// </summary>
[Property] public MediaScreen Screen { get; set; }
MediaPlayer Target => Player.IsValid() ? Player : (Player = GetComponentInParent<MediaPlayer>());
MediaScreen TargetScreen => Screen.IsValid() ? Screen : (Screen = GetComponentInChildren<MediaScreen>());
public bool CanPress( IPressable.Event e ) => Target.IsValid();
public bool Press( IPressable.Event e )
{
if ( !Target.IsValid() ) return false;
// The screen may already have handled this USE press itself this frame
if ( !UseHandled.TryConsume() ) return true;
// Press runs on the client that pressed, so this only affects them (button presses go through the
// player's permission-checked host requests)
var screen = TargetScreen;
if ( screen.IsValid() && screen.TryPressAimedControl() )
return true;
MediaRemote.Toggle( Target );
return true;
}
public IPressable.Tooltip? GetTooltip( IPressable.Event e )
{
if ( !Target.IsValid() ) return null;
var title = Target.HasMedia ? Target.Title : "Media Player";
var icon = Target.AudioOnlyPlayer ? "speaker" : "smart_display";
var action = TargetScreen.IsValid() ? TargetScreen.AimedActionName : null;
if ( action is not null )
return new IPressable.Tooltip( action, "touch_app", title );
return new IPressable.Tooltip( title, icon, "Open controls" );
}
}
Game
library
namespace Bimp.Resolver;
/// <summary>
/// One format an extractor found.
/// </summary>
public sealed class MediaFormat
{
public string Id { get; set; }
public string Url { get; set; }
/// <summary> "webm", "mp4", "mp3"... </summary>
public string Container { get; set; }
/// <summary> Lowercase codec name, null if the format has no video. </summary>
public string VideoCodec { get; set; }
/// <summary> Lowercase codec name, null if the format has no audio. </summary>
public string AudioCodec { get; set; }
public int Height { get; set; }
public float Fps { get; set; }
public long Bitrate { get; set; }
public long Size { get; set; }
public bool Hdr { get; set; }
/// <summary> A dynamic range compressed copy of an audio track. </summary>
public bool Drc { get; set; }
/// <summary> Audio language ("en", "es-419"), null if unlabelled. </summary>
public string Language { get; set; }
public string LanguageName { get; set; }
/// <summary> This is the original audio track. </summary>
public bool LanguageDefault { get; set; }
public bool HasVideo => VideoCodec is not null;
public bool HasAudio => AudioCodec is not null;
public bool IsVp9OrAv1 => VideoCodec is { } v && (v.StartsWith( "vp9" ) || v.StartsWith( "vp09" ) || v.StartsWith( "av01" ) || v.StartsWith( "av1" ));
public bool IsOpus => AudioCodec?.StartsWith( "opus" ) ?? false;
public StreamFile ToFile() => new() { Url = Url, Size = Size, FormatId = Id, Container = Container };
}
/// <summary>
/// Picks formats the engine can play. A port of the old resolver's format picking:
/// VP9/AV1 WebM video + Opus WebM audio (merged locally), else a single muxed file.
/// </summary>
public static class FormatSelector
{
/// <summary> Video-only VP9/AV1 WebM files - the ones we can merge without re-encoding. </summary>
public static List<MediaFormat> WebmVideo( IEnumerable<MediaFormat> formats )
=> formats.Where( f => f.Container == "webm" && f.IsVp9OrAv1 && !f.HasAudio ).ToList();
/// <summary> Audio-only Opus WebM files (every language), without the dynamic-range-compressed copies. </summary>
public static List<MediaFormat> WebmAudio( IEnumerable<MediaFormat> formats )
=> formats.Where( f => f.Container == "webm" && f.IsOpus && !f.HasVideo && !f.Drc ).ToList();
/// <summary>
/// Tallest at or below the limit first, SDR before HDR (the engine's HDR output is untested and would look
/// washed out anyway), then fps, then bitrate.
/// </summary>
static (bool, bool, int, float, long) ScoreVideo( MediaFormat f, int limit )
=> (f.Height <= limit, !f.Hdr, f.Height <= limit ? f.Height : -f.Height, f.Fps, f.Bitrate);
/// <summary>
/// VP9 at 4K and more than 30 fps: more than the engine's VP9 decoder keeps up with (measured 19-45 frames a second
/// on a Ryzen 9 9900X3D; 1440p60 and 4K30 are fine).
/// </summary>
public static bool TooHeavyForVp9( MediaFormat f ) => f.Height >= 2160 && f.Fps > 30;
/// <summary> An 8-bit SDR AV1 video-only MP4 with the same height and frame rate, highest bitrate first. </summary>
public static MediaFormat Av1Like( IEnumerable<MediaFormat> formats, MediaFormat like )
=> formats.Where( f => f.Container == "mp4" && !f.HasAudio && !f.Hdr && f.Height == like.Height && MathF.Abs( f.Fps - like.Fps ) < 1
&& f.VideoCodec is { } v && v.StartsWith( "av01" ) && v.Split( '.' ) is { Length: >= 4 } p && p[3] == "08" )
.OrderByDescending( f => f.Bitrate )
.FirstOrDefault();
/// <summary>
/// What a 4K60 AV1 choice carries on with once YouTube stops serving it: the tallest SDR VP9 below 4K at the same
/// frame rate (1440p60 - the engine's VP9 decoder keeps up with that).
/// </summary>
public static MediaFormat SwitchLike( IEnumerable<MediaFormat> formats, MediaFormat like )
=> WebmVideo( formats ).Where( f => !f.Hdr && f.Height < 2160 && MathF.Abs( f.Fps - like.Fps ) < 1 && f.VideoCodec is { } v && !v.StartsWith( "av01" ) )
.OrderByDescending( f => f.Height ).ThenByDescending( f => f.Bitrate )
.FirstOrDefault();
public static List<int> Qualities( IEnumerable<MediaFormat> formats )
{
var video = WebmVideo( formats );
var sdr = video.Where( f => !f.Hdr ).ToList();
if ( sdr.Count == 0 ) sdr = video;
return sdr.Select( f => f.Height ).Where( h => h > 0 ).Distinct().OrderByDescending( h => h ).ToList();
}
/// <summary> The original audio track's language. </summary>
static string DefaultLanguage( List<MediaFormat> audio )
{
var labelled = audio.Where( f => !string.IsNullOrEmpty( f.Language ) ).ToList();
if ( labelled.Count == 0 ) return null;
return (labelled.FirstOrDefault( f => f.LanguageDefault ) ?? labelled[0]).Language;
}
public static List<MediaAudioTrack> AudioTracks( IEnumerable<MediaFormat> formats )
{
var audio = WebmAudio( formats );
var def = DefaultLanguage( audio );
var tracks = new Dictionary<string, MediaAudioTrack>();
foreach ( var f in audio )
{
if ( string.IsNullOrEmpty( f.Language ) || tracks.ContainsKey( f.Language ) ) continue;
tracks[f.Language] = new MediaAudioTrack { Id = f.Language, Name = f.LanguageName ?? f.Language, IsDefault = f.Language == def };
}
// a single (or unlabelled) track isn't a choice
if ( tracks.Count < 2 ) return new List<MediaAudioTrack>();
return tracks.Values.OrderBy( t => !t.IsDefault ).ThenBy( t => t.Name.ToLowerInvariant() ).ToList();
}
/// <summary> Do these formats have an audio track in this language, matched the way <see cref="PickAudio"/> matches it? </summary>
public static bool HasLanguage( IEnumerable<MediaFormat> formats, string lang )
{
if ( string.IsNullOrWhiteSpace( lang ) ) return true;
var primary = lang.Split( '-' )[0];
return WebmAudio( formats ).Any( f => string.Equals( f.Language, lang, StringComparison.OrdinalIgnoreCase )
|| string.Equals( (f.Language ?? "").Split( '-' )[0], primary, StringComparison.OrdinalIgnoreCase ) );
}
/// <summary> Audio formats for the requested language ("es" finds "es-419"), falling back to the original track. </summary>
static List<MediaFormat> PickAudio( List<MediaFormat> audio, string lang )
{
if ( audio.Count == 0 ) return audio;
if ( !string.IsNullOrWhiteSpace( lang ) )
{
var exact = audio.Where( f => string.Equals( f.Language, lang, StringComparison.OrdinalIgnoreCase ) ).ToList();
if ( exact.Count > 0 ) return exact;
var primary = lang.Split( '-' )[0];
var prefix = audio.Where( f => string.Equals( (f.Language ?? "").Split( '-' )[0], primary, StringComparison.OrdinalIgnoreCase ) ).ToList();
if ( prefix.Count > 0 ) return prefix;
}
var def = DefaultLanguage( audio );
return def is null ? audio : audio.Where( f => f.Language == def ).ToList();
}
/// <summary>
/// Best first. With an exact height, that height wins if it exists, else the nearest below it.
/// </summary>
static List<MediaFormat> PickVideo( List<MediaFormat> video, int maxHeight, int height )
{
var limit = height > 0 ? height : maxHeight;
return video.OrderByDescending( f => ScoreVideo( f, limit ) ).ToList();
}
public static StreamPlan Plan( IReadOnlyList<MediaFormat> formats, StreamRequest request )
{
var maxh = Math.Clamp( request.MaxHeight, 144, 2160 );
var audio = PickAudio( WebmAudio( formats ), request.Language ).OrderByDescending( f => f.Bitrate ).ToList();
if ( request.AudioOnly )
{
if ( audio.Count > 0 )
return new StreamPlan { Kind = StreamKind.Audio, Audio = audio[0].ToFile() };
// anything with audio the engine can decode on its own
var anyAudio = formats.Where( f => f.HasAudio && !f.HasVideo && f.Container is "mp3" or "ogg" or "m4a" )
.OrderByDescending( f => f.Bitrate ).FirstOrDefault()
?? formats.Where( f => f.HasAudio && f.Container is "mp4" or "webm" ).OrderBy( f => f.Height ).FirstOrDefault();
if ( anyAudio is not null )
return new StreamPlan { Kind = StreamKind.Direct, DirectUrl = anyAudio.Url };
throw new ResolveException( "No playable audio found." );
}
var video = PickVideo( WebmVideo( formats ), maxh, request.Height );
if ( video.Count > 0 && audio.Count > 0 )
return new StreamPlan { Kind = StreamKind.Merge, Video = video[0].ToFile(), Audio = audio[0].ToFile() };
// a single muxed file the engine can decode (VP9/AV1+Opus WebM, or H.264/AAC MP4)
var muxed = formats.Where( f => f.HasVideo && f.HasAudio && f.Container is "webm" or "mp4" )
.OrderByDescending( f => ScoreVideo( f, maxh ) )
.ThenByDescending( f => f.Container == "webm" )
.FirstOrDefault();
if ( muxed is not null )
return new StreamPlan { Kind = StreamKind.Direct, DirectUrl = muxed.Url };
throw new ResolveException( "No playable formats found." );
}
}
Debug: View Raw JSON Response
{
"TotalCount": 124,
"Files": [
{
"Ident": "rexi.bimp",
"Path": "InputGlyph.cs",
"FileName": "InputGlyph.cs",
"PackageType": "library",
"CodeKind": "Game",
"AssetVersionId": 399292,
"IsPrivate": false,
"Code": "using System;\nusing Sandbox.UI;\n\nnamespace Bimp;\n\n/// <summary>\n/// The button for an input action, drawn the way the player's device shows it: a key cap on keyboard, the pad's own\n/// button (Xbox, PlayStation, Nintendo...) on a controller. It follows them live as they switch device. Sized to the\n/// text around it (1.25em tall, as wide as the glyph).\n/// </summary>\n[StyleSheet.Inline( \"bimp-inputglyph\", Styles )]\npublic sealed class InputGlyph : Panel\n{\n\tconst string Styles = \"\"\"\n\t\t.inputglyph { height: 1.25em; aspect-ratio: 1; flex-shrink: 0; background-size: contain; background-repeat: no-repeat; background-position: center; }\n\t\t\"\"\";\n\n\t/// <summary> The input action to show the button of, e.g. \"use\". </summary>\n\tpublic string Action { get; set; }\n\n\t/// <summary> An analog input to show instead of an action, by <see cref=\"InputAnalog\"/> name, e.g. \"LeftStickY\". </summary>\n\tpublic string Analog { get; set; }\n\n\t/// <summary> A keyboard key to show instead of an action, e.g. \"escape\" (keyboard glyph whatever the device). </summary>\n\tpublic string Key { get; set; }\n\n\t/// <summary> Glyph resolution. Medium (128px) stays sharp on the big world screens. </summary>\n\tpublic InputGlyphSize Size { get; set; } = InputGlyphSize.Medium;\n\n\tTexture shown;\n\n\tpublic InputGlyph()\n\t{\n\t\tAddClass( \"inputglyph\" );\n\t}\n\n\tpublic override void Tick()\n\t{\n\t\tbase.Tick();\n\n\t\t// cheap, and the device can change any frame (the engine's own advice for glyphs)\n\t\tvar texture = Glyph();\n\t\tif ( texture == shown ) return;\n\n\t\tshown = texture;\n\t\tStyle.SetBackgroundImage( texture );\n\t\tStyle.AspectRatio = texture is { Height: > 0 } ? texture.Width / (float)texture.Height : 1;\n\t}\n\n\tTexture Glyph()\n\t{\n\t\tif ( !string.IsNullOrWhiteSpace( Key ) ) return Input.Keyboard.GetGlyph( Key, Size );\n\t\tif ( Enum.TryParse<InputAnalog>( Analog, true, out var analog ) ) return Input.GetGlyph( analog, Size );\n\t\tif ( !string.IsNullOrWhiteSpace( Action ) ) return Input.GetGlyph( Action, Size, false );\n\t\treturn null;\n\t}\n}\n"
},
{
"Ident": "rexi.bimp",
"Path": "Resolver/Extractors/DashExtractor.cs",
"FileName": "DashExtractor.cs",
"PackageType": "library",
"CodeKind": "Game",
"AssetVersionId": 399292,
"IsPrivate": false,
"Code": "using System.Threading;\nusing Bimp.Resolver.Live;\n\nnamespace Bimp.Resolver.Extractors;\n\n/// <summary>\n/// Any .mpd link: a finished (static) DASH manifest plays as a seekable video, remuxed from a start time like a finished\n/// HLS playlist (see <see cref=\"MpdPlaylists\"/>, which turns the manifest into playlists <see cref=\"HlsReader\"/> reads).\n/// </summary>\npublic sealed class DashExtractor : IExtractor\n{\n\tpublic string Key => \"dash\";\n\n\tpublic bool CanHandle( Uri url ) => MpdPlaylists.IsMpd( url );\n\n\tpublic async Task<MediaInfo> GetInfoAsync( Uri url, bool audioOnly, CancellationToken ct )\n\t{\n\t\tvar mpd = await MpdPlaylists.LoadAsync( url, ct );\n\t\tvar name = Uri.UnescapeDataString( System.IO.Path.GetFileNameWithoutExtension( url.AbsolutePath ) );\n\n\t\treturn new MediaInfo\n\t\t{\n\t\t\tExtractor = Key,\n\t\t\tId = url.ToString(),\n\t\t\tTitle = !string.IsNullOrWhiteSpace( mpd.Title ) ? mpd.Title : name is \"manifest\" or \"index\" or \"stream\" or \"\" ? url.Host : $\"{url.Host} - {name}\",\n\t\t\tDuration = (float)mpd.Duration,\n\t\t\tAudioOnly = audioOnly || mpd.Heights.Count == 0,\n\t\t\tMerged = true,\n\t\t\tQualities = audioOnly || mpd.Heights.Count < 2 ? new List<int>() : mpd.Heights,\n\t\t};\n\t}\n\n\tpublic Task<StreamPlan> GetStreamsAsync( string id, StreamRequest request, CancellationToken ct )\n\t\t=> Task.FromResult( new StreamPlan { Kind = StreamKind.Hls, DirectUrl = id } );\n}\n"
},
{
"Ident": "rexi.bimp",
"Path": "Resolver/Live/Mp4Writer.cs",
"FileName": "Mp4Writer.cs",
"PackageType": "library",
"CodeKind": "Game",
"AssetVersionId": 399292,
"IsPrivate": false,
"Code": "using System.Buffers.Binary;\r\nusing System.IO;\r\nusing System.Text;\r\n\r\nnamespace Bimp.Resolver.Live;\r\n\r\n/// <summary> One video access unit (timestamps in 90 kHz units). </summary>\r\npublic sealed class VideoSample\r\n{\r\n\tpublic long Dts;\r\n\tpublic long Pts;\r\n\tpublic bool Key;\r\n\t/// <summary> H.264: AVCC (NAL units with 4 byte lengths). AV1: a temporal unit of sized OBUs. </summary>\r\n\tpublic byte[] Data;\r\n}\r\n\r\n/// <summary> A video track's codec and decoder configuration: what its MP4 sample entry says. </summary>\r\npublic sealed class VideoFormat\r\n{\r\n\tpublic string Codec { get; private init; }\r\n\tpublic int Width { get; private init; }\r\n\tpublic int Height { get; private init; }\r\n\r\n\t/// <summary> The sample entry's four character code and its configuration box. </summary>\r\n\tinternal string Fourcc { get; private init; }\r\n\tinternal string ConfigBox { get; private init; }\r\n\tinternal byte[] Config { get; private init; }\r\n\r\n\t/// <summary>\r\n\t/// The engine's H.264 decoder (Media Foundation) holds back the last 22 frames of a file; its AV1 decoder shows\r\n\t/// every frame (see <see cref=\"Media.Av1\"/>).\r\n\t/// </summary>\r\n\tpublic bool HoldsBackFrames => Fourcc == \"avc1\";\r\n\r\n\tpublic bool IsH264 => Fourcc == \"avc1\";\r\n\r\n\tpublic static VideoFormat H264( byte[] sps, byte[] pps )\r\n\t{\r\n\t\tvar info = Live.H264.ParseSps( sps );\r\n\t\treturn new VideoFormat { Codec = \"H.264\", Width = info.Width, Height = info.Height, Fourcc = \"avc1\", ConfigBox = \"avcC\", Config = Live.H264.AvcConfig( sps, pps ) };\r\n\t}\r\n\r\n\tpublic static VideoFormat Av1( Media.Av1.SequenceHeader seq )\r\n\t\t=> new() { Codec = \"AV1\", Width = seq.Width, Height = seq.Height, Fourcc = \"av01\", ConfigBox = \"av1C\", Config = Media.Av1.ConfigRecord( seq ) };\r\n\r\n\tpublic override string ToString() => $\"{Codec} {Width}x{Height}\";\r\n}\r\n\r\n/// <summary> One raw AAC frame (no ADTS header), 1024 samples. Pts in 90 kHz units. </summary>\r\npublic sealed class AudioSample\r\n{\r\n\tpublic long Pts;\r\n\tpublic byte[] Data;\r\n}\r\n\r\n/// <summary> An AAC track's decoder configuration. </summary>\r\npublic sealed class AacConfig\r\n{\r\n\t/// <summary> AudioSpecificConfig. </summary>\r\n\tpublic byte[] Asc;\r\n\tpublic int SampleRate;\r\n\tpublic int Channels;\r\n\r\n\tstatic readonly int[] Rates = { 96000, 88200, 64000, 48000, 44100, 32000, 24000, 22050, 16000, 12000, 11025, 8000, 7350 };\r\n\r\n\tpublic static int RateIndex( int rate ) => Array.IndexOf( Rates, rate );\r\n\tpublic static int RateOf( int index ) => index >= 0 && index < Rates.Length ? Rates[index] : 0;\r\n\r\n\t/// <summary> Build from ADTS header fields. </summary>\r\n\tpublic static AacConfig FromAdts( int profile, int rateIndex, int channelConfig )\r\n\t{\r\n\t\tvar objectType = profile + 1;\r\n\t\tvar asc = new byte[]\r\n\t\t{\r\n\t\t\t(byte)((objectType << 3) | (rateIndex >> 1)),\r\n\t\t\t(byte)(((rateIndex & 1) << 7) | (channelConfig << 3)),\r\n\t\t};\r\n\t\treturn new AacConfig { Asc = asc, SampleRate = RateOf( rateIndex ), Channels = channelConfig == 7 ? 8 : channelConfig };\r\n\t}\r\n\r\n\t/// <summary> Parse an AudioSpecificConfig (from SDP \"config=\" or an esds). </summary>\r\n\tpublic static AacConfig FromAsc( byte[] asc )\r\n\t{\r\n\t\tif ( asc is null || asc.Length < 2 ) return null;\r\n\t\tvar v = (asc[0] << 8) | asc[1];\r\n\t\tvar rateIndex = (v >> 7) & 0xF;\r\n\t\tvar channelConfig = (v >> 3) & 0xF;\r\n\t\treturn new AacConfig { Asc = asc, SampleRate = RateOf( rateIndex ), Channels = channelConfig == 7 ? 8 : Math.Max( 1, channelConfig ) };\r\n\t}\r\n\r\n\tpublic bool SameAs( AacConfig o ) => o is not null && Asc.AsSpan().SequenceEqual( o.Asc );\r\n\r\n\t/// <summary>\r\n\t/// The rate the mp4a sample entry declares. The engine's AAC decoder (Media Foundation) won't open audio below\r\n\t/// 32 kHz - measured 2026-10-01: 16, 22.05 and 24 kHz AAC-LC play nothing, 32/44.1/48 kHz play - and an HE-AAC\r\n\t/// stream's core (all the ASC says when the SBR is implicit, as from ADTS) is half its real rate. Declaring twice\r\n\t/// the core is how Media Foundation is told the stream is HE-AAC: it then opens it at the doubled rate.\r\n\t/// </summary>\r\n\tpublic int OutputRate => SampleRate is > 0 and < 32000 ? SampleRate * 2 : SampleRate;\r\n}\r\n\r\n/// <summary>\r\n/// Writes a finished, moov-first MP4 (H.264 and/or AAC) - the only kind of file the engine opens instantly\r\n/// (it walks every fragment of a fragmented MP4 on open, and needs the index before the data).\r\n/// The file's timeline starts at <see cref=\"Write\"/>'s <c>zero</c>.\r\n/// </summary>\r\npublic static class Mp4Writer\r\n{\r\n\tconst uint MovieTimescale = 90000;\r\n\r\n\tpublic static void Write( Stream output, long zero,\r\n\t\tList<VideoSample> video, byte[] sps, byte[] pps,\r\n\t\tList<AudioSample> audio, AacConfig aac )\r\n\t\t=> Write( output, zero, video, sps is null || pps is null ? null : VideoFormat.H264( sps, pps ), audio, aac );\r\n\r\n\tpublic static void Write( Stream output, long zero,\r\n\t\tList<VideoSample> video, VideoFormat format,\r\n\t\tList<AudioSample> audio, AacConfig aac )\r\n\t{\r\n\t\tvar hasVideo = video is { Count: > 0 } && format is not null;\r\n\t\tvar hasAudio = audio is { Count: > 0 } && aac is not null && aac.SampleRate > 0;\r\n\r\n\t\tvar ftyp = Box( \"ftyp\", Concat( Ascii( \"isom\" ), U32( 512 ), Ascii( \"isom\" ), Ascii( \"iso2\" ), Ascii( format?.Fourcc ?? \"avc1\" ), Ascii( \"mp41\" ) ) );\r\n\r\n\t\t// sample data: all video, then all audio (one chunk each - it's a local file, no need to interleave)\r\n\t\tvar videoData = hasVideo ? video.Select( s => s.Data ).ToList() : new List<byte[]>();\r\n\t\tvar audioData = hasAudio ? audio.Select( s => s.Data ).ToList() : new List<byte[]>();\r\n\t\tlong videoBytes = videoData.Sum( d => (long)d.Length );\r\n\r\n\t\t// moov has a fixed size whatever the offsets are, so build it once to measure, then for real\r\n\t\tbyte[] Moov( uint videoOffset, uint audioOffset )\r\n\t\t{\r\n\t\t\tvar traks = new List<byte[]>();\r\n\t\t\tuint id = 1;\r\n\t\t\tif ( hasVideo ) traks.Add( VideoTrak( id++, zero, video, format, videoOffset ) );\r\n\t\t\tif ( hasAudio ) traks.Add( AudioTrak( id++, zero, audio, aac, audioOffset ) );\r\n\t\t\tvar duration = Math.Max( hasVideo ? VideoDuration( video, zero ) : 0, hasAudio ? AudioMovieDuration( audio, aac, zero ) : 0 );\r\n\t\t\treturn Box( \"moov\", Concat( new[] { Mvhd( duration, id ) }.Concat( traks ).ToArray() ) );\r\n\t\t}\r\n\r\n\t\tvar moovSize = Moov( 0, 0 ).Length;\r\n\t\tvar dataStart = (uint)(ftyp.Length + moovSize + 8);\r\n\t\tvar moov = Moov( dataStart, (uint)(dataStart + videoBytes) );\r\n\r\n\t\tvar mdatSize = 8 + videoBytes + audioData.Sum( d => (long)d.Length );\r\n\t\toutput.Write( ftyp );\r\n\t\toutput.Write( moov );\r\n\t\toutput.Write( U32( (uint)mdatSize ) );\r\n\t\toutput.Write( Ascii( \"mdat\" ) );\r\n\t\tforeach ( var d in videoData ) output.Write( d );\r\n\t\tforeach ( var d in audioData ) output.Write( d );\r\n\t}\r\n\r\n\tstatic long VideoDuration( List<VideoSample> v, long zero )\r\n\t{\r\n\t\tvar last = v[^1];\r\n\t\tvar frame = v.Count > 1 ? Math.Max( 1, last.Dts - v[^2].Dts ) : 3000;\r\n\t\treturn last.Dts + frame - zero;\r\n\t}\r\n\r\n\tstatic long AudioMovieDuration( List<AudioSample> a, AacConfig aac, long zero )\r\n\t\t=> Math.Max( 0, a[0].Pts - zero ) + (long)a.Count * 1024 * MovieTimescale / aac.SampleRate;\r\n\r\n\tstatic byte[] VideoTrak( uint id, long zero, List<VideoSample> v, VideoFormat info, uint offset )\r\n\t{\r\n\t\t// durations between decode times; the last frame lasts as long as the one before it\r\n\t\tvar durations = new List<uint>();\r\n\t\tfor ( int i = 0; i < v.Count; i++ )\r\n\t\t{\r\n\t\t\tvar d = i + 1 < v.Count ? v[i + 1].Dts - v[i].Dts : (v.Count > 1 ? v[i].Dts - v[i - 1].Dts : 3000);\r\n\t\t\tdurations.Add( (uint)Math.Clamp( d, 1, 90000 ) );\r\n\t\t}\r\n\t\tvar mediaDuration = durations.Sum( d => (long)d );\r\n\t\tvar offsets = v.Select( s => s.Pts - s.Dts ).ToList();\r\n\t\tvar hasCtts = offsets.Any( o => o != 0 );\r\n\r\n\t\tvar stbl = new List<byte[]>\r\n\t\t{\r\n\t\t\tBox( \"stsd\", Concat( U32( 0 ), U32( 1 ), VisualEntry( info ) ) ),\r\n\t\t\tStts( durations ),\r\n\t\t};\r\n\t\tif ( hasCtts ) stbl.Add( Ctts( offsets ) );\r\n\t\tstbl.Add( Stss( v ) );\r\n\t\tstbl.Add( Stsc( v.Count ) );\r\n\t\tstbl.Add( Stsz( v.Select( s => s.Data.Length ) ) );\r\n\t\tstbl.Add( Box( \"stco\", Concat( U32( 0 ), U32( 1 ), U32( offset ) ) ) );\r\n\r\n\t\t// the first frame shows at its presentation time; start the track there\r\n\t\tvar elst = Elst( Math.Max( 0, v[0].Dts - zero ), mediaDuration, Math.Max( 0, offsets[0] ) );\r\n\r\n\t\treturn Box( \"trak\", Concat(\r\n\t\t\tTkhd( id, mediaDuration + Math.Max( 0, v[0].Dts - zero ), false, info.Width, info.Height ),\r\n\t\t\tBox( \"edts\", elst ),\r\n\t\t\tBox( \"mdia\", Concat(\r\n\t\t\t\tMdhd( 90000, mediaDuration ),\r\n\t\t\t\tHdlr( \"vide\" ),\r\n\t\t\t\tBox( \"minf\", Concat(\r\n\t\t\t\t\tFullBox( \"vmhd\", 0, 1, new byte[8] ),\r\n\t\t\t\t\tDinf(),\r\n\t\t\t\t\tBox( \"stbl\", Concat( stbl.ToArray() ) ) ) ) ) ) ) );\r\n\t}\r\n\r\n\tstatic byte[] AudioTrak( uint id, long zero, List<AudioSample> a, AacConfig aac, uint offset )\r\n\t{\r\n\t\tvar mediaDuration = (long)a.Count * 1024;\r\n\t\tvar startDelay = Math.Max( 0, a[0].Pts - zero ); // movie timescale\r\n\r\n\t\tvar stbl = Concat(\r\n\t\t\tBox( \"stsd\", Concat( U32( 0 ), U32( 1 ), Mp4a( aac ) ) ),\r\n\t\t\tStts( Enumerable.Repeat( 1024u, a.Count ).ToList() ),\r\n\t\t\tStsc( a.Count ),\r\n\t\t\tStsz( a.Select( s => s.Data.Length ) ),\r\n\t\t\tBox( \"stco\", Concat( U32( 0 ), U32( 1 ), U32( offset ) ) ) );\r\n\r\n\t\tvar movieDuration = mediaDuration * MovieTimescale / aac.SampleRate;\r\n\r\n\t\treturn Box( \"trak\", Concat(\r\n\t\t\tTkhd( id, startDelay + movieDuration, true, 0, 0 ),\r\n\t\t\tBox( \"edts\", Elst( startDelay, movieDuration, 0 ) ),\r\n\t\t\tBox( \"mdia\", Concat(\r\n\t\t\t\tMdhd( (uint)aac.SampleRate, mediaDuration ),\r\n\t\t\t\tHdlr( \"soun\" ),\r\n\t\t\t\tBox( \"minf\", Concat(\r\n\t\t\t\t\tFullBox( \"smhd\", 0, 0, new byte[4] ),\r\n\t\t\t\t\tDinf(),\r\n\t\t\t\t\tBox( \"stbl\", stbl ) ) ) ) ) ) );\r\n\t}\r\n\r\n\t/// <summary> An edit list: an optional empty edit (delay), then the media from mediaTime. </summary>\r\n\tstatic byte[] Elst( long delay, long movieDuration, long mediaTime )\r\n\t{\r\n\t\tvar entries = new List<byte[]>();\r\n\t\tif ( delay > 0 ) entries.Add( Concat( U32( (uint)delay ), U32( 0xFFFFFFFF ), U32( 0x00010000 ) ) );\r\n\t\tentries.Add( Concat( U32( (uint)movieDuration ), U32( (uint)mediaTime ), U32( 0x00010000 ) ) );\r\n\t\treturn FullBox( \"elst\", 0, 0, Concat( new[] { U32( (uint)entries.Count ) }.Concat( entries ).ToArray() ) );\r\n\t}\r\n\r\n\tstatic byte[] Mvhd( long duration, uint nextTrack )\r\n\t{\r\n\t\treturn FullBox( \"mvhd\", 0, 0, Concat(\r\n\t\t\tU32( 0 ), U32( 0 ), U32( MovieTimescale ), U32( (uint)duration ),\r\n\t\t\tU32( 0x00010000 ), U16( 0x0100 ), new byte[10],\r\n\t\t\tMatrix(), new byte[24], U32( nextTrack ) ) );\r\n\t}\r\n\r\n\tstatic byte[] Tkhd( uint id, long duration, bool audio, int width, int height )\r\n\t{\r\n\t\treturn FullBox( \"tkhd\", 0, 3, Concat(\r\n\t\t\tU32( 0 ), U32( 0 ), U32( id ), U32( 0 ), U32( (uint)duration ),\r\n\t\t\tnew byte[8], U16( 0 ), U16( 0 ), U16( (ushort)(audio ? 0x0100 : 0) ), U16( 0 ),\r\n\t\t\tMatrix(), U32( (uint)width << 16 ), U32( (uint)height << 16 ) ) );\r\n\t}\r\n\r\n\tstatic byte[] Mdhd( uint timescale, long duration )\r\n\t\t=> FullBox( \"mdhd\", 0, 0, Concat( U32( 0 ), U32( 0 ), U32( timescale ), U32( (uint)duration ), U16( 0x55C4 ), U16( 0 ) ) );\r\n\r\n\tstatic byte[] Hdlr( string type )\r\n\t\t=> FullBox( \"hdlr\", 0, 0, Concat( U32( 0 ), Ascii( type ), new byte[12], Encoding.ASCII.GetBytes( \"bimp\\0\" ) ) );\r\n\r\n\tstatic byte[] Dinf() => Box( \"dinf\", FullBox( \"dref\", 0, 0, Concat( U32( 1 ), FullBox( \"url \", 0, 1, Array.Empty<byte>() ) ) ) );\r\n\r\n\t/// <summary> avc1 + avcC, or av01 + av1C. </summary>\r\n\tstatic byte[] VisualEntry( VideoFormat f )\r\n\t{\r\n\t\treturn Box( f.Fourcc, Concat(\r\n\t\t\tnew byte[6], U16( 1 ), // reserved, data_reference_index\r\n\t\t\tnew byte[16], // pre_defined, reserved, pre_defined[3]\r\n\t\t\tU16( (ushort)f.Width ), U16( (ushort)f.Height ),\r\n\t\t\tU32( 0x00480000 ), U32( 0x00480000 ), U32( 0 ), U16( 1 ),\r\n\t\t\tnew byte[32], // compressorname\r\n\t\t\tU16( 0x0018 ), U16( 0xFFFF ),\r\n\t\t\tBox( f.ConfigBox, f.Config ) ) );\r\n\t}\r\n\r\n\tstatic byte[] Mp4a( AacConfig aac )\r\n\t{\r\n\t\t// ES_Descriptor > DecoderConfigDescriptor (AAC, audio stream) > DecoderSpecificInfo (the ASC), SLConfig\r\n\t\tvar dsi = Descriptor( 5, aac.Asc );\r\n\t\tvar dcd = Descriptor( 4, Concat( new byte[] { 0x40, 0x15, 0, 0, 0 }, U32( 0 ), U32( 0 ), dsi ) );\r\n\t\tvar es = Descriptor( 3, Concat( U16( 1 ), new byte[] { 0 }, dcd, Descriptor( 6, new byte[] { 2 } ) ) );\r\n\r\n\t\t// channelcount must match the ASC - the engine trusts this field (see Mp4ChannelFix)\r\n\t\treturn Box( \"mp4a\", Concat(\r\n\t\t\tnew byte[6], U16( 1 ), new byte[8],\r\n\t\t\tU16( (ushort)aac.Channels ), U16( 16 ), U16( 0 ), U16( 0 ),\r\n\t\t\tU32( (uint)Math.Min( aac.OutputRate, 65535 ) << 16 ),\r\n\t\t\tFullBox( \"esds\", 0, 0, es ) ) );\r\n\t}\r\n\r\n\tstatic byte[] Descriptor( byte tag, byte[] body )\r\n\t{\r\n\t\t// 4 byte length form, like most muxers write it\r\n\t\tvar len = body.Length;\r\n\t\treturn Concat( new byte[] { tag, (byte)(0x80 | (len >> 21) & 0x7F), (byte)(0x80 | (len >> 14) & 0x7F), (byte)(0x80 | (len >> 7) & 0x7F), (byte)(len & 0x7F) }, body );\r\n\t}\r\n\r\n\tstatic byte[] Stts( List<uint> durations )\r\n\t{\r\n\t\tvar runs = new List<(uint count, uint delta)>();\r\n\t\tforeach ( var d in durations )\r\n\t\t{\r\n\t\t\tif ( runs.Count > 0 && runs[^1].delta == d ) runs[^1] = (runs[^1].count + 1, d);\r\n\t\t\telse runs.Add( (1, d) );\r\n\t\t}\r\n\t\treturn FullBox( \"stts\", 0, 0, Concat( new[] { U32( (uint)runs.Count ) }.Concat( runs.Select( r => Concat( U32( r.count ), U32( r.delta ) ) ) ).ToArray() ) );\r\n\t}\r\n\r\n\tstatic byte[] Ctts( List<long> offsets )\r\n\t{\r\n\t\t// version 0 (unsigned) like normal muxers write it - B-frame offsets are never negative when decode time is the\r\n\t\t// earliest presentation; version 1 (signed) only if we really need it\r\n\t\tvar signed = offsets.Any( o => o < 0 );\r\n\t\tvar runs = new List<(uint count, long offset)>();\r\n\t\tforeach ( var o in offsets )\r\n\t\t{\r\n\t\t\tif ( runs.Count > 0 && runs[^1].offset == o ) runs[^1] = (runs[^1].count + 1, o);\r\n\t\t\telse runs.Add( (1, o) );\r\n\t\t}\r\n\t\treturn FullBox( \"ctts\", (byte)(signed ? 1 : 0), 0, Concat( new[] { U32( (uint)runs.Count ) }.Concat( runs.Select( r => Concat( U32( r.count ), U32( unchecked((uint)(int)r.offset) ) ) ) ).ToArray() ) );\r\n\t}\r\n\r\n\tstatic byte[] Stss( List<VideoSample> v )\r\n\t{\r\n\t\tvar keys = v.Select( ( s, i ) => (s, i) ).Where( x => x.s.Key ).Select( x => U32( (uint)x.i + 1 ) ).ToList();\r\n\t\treturn FullBox( \"stss\", 0, 0, Concat( new[] { U32( (uint)keys.Count ) }.Concat( keys ).ToArray() ) );\r\n\t}\r\n\r\n\tstatic byte[] Stsc( int samples ) => FullBox( \"stsc\", 0, 0, Concat( U32( 1 ), U32( 1 ), U32( (uint)samples ), U32( 1 ) ) );\r\n\r\n\tstatic byte[] Stsz( IEnumerable<int> sizes )\r\n\t{\r\n\t\tvar list = sizes.ToList();\r\n\t\treturn FullBox( \"stsz\", 0, 0, Concat( new[] { U32( 0 ), U32( (uint)list.Count ) }.Concat( list.Select( s => U32( (uint)s ) ) ).ToArray() ) );\r\n\t}\r\n\r\n\tstatic byte[] Matrix() => Concat( U32( 0x00010000 ), U32( 0 ), U32( 0 ), U32( 0 ), U32( 0x00010000 ), U32( 0 ), U32( 0 ), U32( 0 ), U32( 0x40000000 ) );\r\n\r\n\tstatic byte[] Box( string type, byte[] body ) => Concat( U32( (uint)(8 + body.Length) ), Ascii( type ), body );\r\n\r\n\tstatic byte[] FullBox( string type, byte version, uint flags, byte[] body )\r\n\t\t=> Box( type, Concat( new[] { version, (byte)(flags >> 16), (byte)(flags >> 8), (byte)flags }, body ) );\r\n\r\n\tstatic byte[] Ascii( string s ) => Encoding.ASCII.GetBytes( s );\r\n\r\n\tstatic byte[] U32( uint v ) { var b = new byte[4]; BinaryPrimitives.WriteUInt32BigEndian( b, v ); return b; }\r\n\tstatic byte[] U16( ushort v ) { var b = new byte[2]; BinaryPrimitives.WriteUInt16BigEndian( b, v ); return b; }\r\n\r\n\tstatic byte[] Concat( params byte[][] parts ) => Media.Ebml.Concat( parts );\r\n}\r\n"
},
{
"Ident": "rexi.bimp",
"Path": "Resolver/Media/Fmp4Reader.cs",
"FileName": "Fmp4Reader.cs",
"PackageType": "library",
"CodeKind": "Game",
"AssetVersionId": 399292,
"IsPrivate": false,
"Code": "using System.Buffers.Binary;\nusing System.Text;\n\nnamespace Bimp.Resolver.Media;\n\n/// <summary>\n/// Reads fragmented MP4 (the CMAF / DASH / HLS fMP4 layout): the tracks in an init segment's moov, and the samples in\n/// moof + mdat fragments. Shared by YouTube's AV1 files (<see cref=\"WebmSource.ProbeMp4Async\"/>) and fMP4 HLS.\n/// </summary>\npublic static class Fmp4Reader\n{\n\t/// <summary> One track of an init segment. </summary>\n\tpublic sealed class Track\n\t{\n\t\tpublic uint Id;\n\n\t\t/// <summary> \"vide\" or \"soun\". </summary>\n\t\tpublic string Handler;\n\t\tpublic long Timescale;\n\t\tpublic double DurationSeconds;\n\n\t\t/// <summary> The sample entry's four character code: avc1, avc3, hvc1, hev1, av01, mp4a, Opus... </summary>\n\t\tpublic string Codec;\n\n\t\t/// <summary> avcC / av1C body, or the AAC AudioSpecificConfig from the esds. </summary>\n\t\tpublic byte[] Config;\n\t\tpublic int Width, Height;\n\t\tpublic int Channels, SampleRate;\n\n\t\t// trex defaults, for fragments that don't set their own\n\t\tpublic uint DefaultDuration, DefaultSize, DefaultFlags;\n\n\t\tpublic bool IsVideo => Handler == \"vide\";\n\t\tpublic bool IsAudio => Handler == \"soun\";\n\t}\n\n\t/// <summary> One sample of a fragment: where its data is in the buffer, and its times in the track's timescale. </summary>\n\tpublic readonly record struct Sample( uint Track, long DecodeTime, int CompositionOffset, uint Duration, bool Key, int Offset, int Size );\n\n\t/// <summary> The tracks of the moov in d[start..end] (an init segment, or the head of a DASH file). </summary>\n\tpublic static List<Track> ReadTracks( byte[] d, int start, int end )\n\t{\n\t\tvar moov = Find( d, start, end, \"moov\" ) ?? throw new MergeException( \"no moov in the mp4\" );\n\t\tvar tracks = new List<Track>();\n\t\tvar trex = new Dictionary<uint, (uint duration, uint size, uint flags)>();\n\t\tforeach ( var (at, size, type, h) in Children( d, moov.at + moov.h, moov.at + moov.size ) )\n\t\t{\n\t\t\tif ( type == \"trak\" && ReadTrak( d, at, at + size ) is { } t ) tracks.Add( t );\n\t\t\telse if ( type == \"mvex\" )\n\t\t\t{\n\t\t\t\tforeach ( var (x, xs, xt, xh) in Children( d, at + h, at + size ) )\n\t\t\t\t{\n\t\t\t\t\tif ( xt != \"trex\" ) continue;\n\t\t\t\t\t// full box: version/flags, track_ID, description index, duration, size, flags\n\t\t\t\t\tvar p = x + xh + 4;\n\t\t\t\t\ttrex[BinaryPrimitives.ReadUInt32BigEndian( d.AsSpan( p ) )] = (\n\t\t\t\t\t\tBinaryPrimitives.ReadUInt32BigEndian( d.AsSpan( p + 8 ) ),\n\t\t\t\t\t\tBinaryPrimitives.ReadUInt32BigEndian( d.AsSpan( p + 12 ) ),\n\t\t\t\t\t\tBinaryPrimitives.ReadUInt32BigEndian( d.AsSpan( p + 16 ) ));\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tforeach ( var t in tracks )\n\t\t{\n\t\t\tif ( trex.TryGetValue( t.Id, out var x ) ) (t.DefaultDuration, t.DefaultSize, t.DefaultFlags) = x;\n\t\t}\n\t\treturn tracks;\n\t}\n\n\tstatic Track ReadTrak( byte[] d, int trak, int end )\n\t{\n\t\tvar t = new Track();\n\t\tif ( Path( d, trak, end, \"tkhd\" ) is { } tkhd )\n\t\t{\n\t\t\tvar p = tkhd.at + tkhd.h;\n\t\t\tt.Id = BinaryPrimitives.ReadUInt32BigEndian( d.AsSpan( p + (d[p] == 1 ? 20 : 12) ) );\n\t\t}\n\n\t\tvar mdhd = Path( d, trak, end, \"mdia\", \"mdhd\" );\n\t\tif ( mdhd is null ) return null;\n\t\tvar mp = mdhd.Value.at + mdhd.Value.h;\n\t\tlong duration;\n\t\tif ( d[mp] == 1 )\n\t\t{\n\t\t\tt.Timescale = BinaryPrimitives.ReadUInt32BigEndian( d.AsSpan( mp + 20 ) );\n\t\t\tduration = (long)BinaryPrimitives.ReadUInt64BigEndian( d.AsSpan( mp + 24 ) );\n\t\t}\n\t\telse\n\t\t{\n\t\t\tt.Timescale = BinaryPrimitives.ReadUInt32BigEndian( d.AsSpan( mp + 12 ) );\n\t\t\tduration = BinaryPrimitives.ReadUInt32BigEndian( d.AsSpan( mp + 16 ) );\n\t\t}\n\t\tif ( t.Timescale <= 0 ) return null;\n\t\tt.DurationSeconds = duration / (double)t.Timescale;\n\n\t\tif ( Path( d, trak, end, \"mdia\", \"hdlr\" ) is { } hdlr ) t.Handler = Encoding.ASCII.GetString( d, hdlr.at + hdlr.h + 8, 4 );\n\n\t\tvar stsd = Path( d, trak, end, \"mdia\", \"minf\", \"stbl\", \"stsd\" );\n\t\tif ( stsd is null ) return t;\n\t\tvar entry = stsd.Value.at + stsd.Value.h + 8; // version/flags, entry count\n\t\tvar (entrySize, codec, entryHead) = Box( d, entry, stsd.Value.at + stsd.Value.size - entry );\n\t\tt.Codec = codec;\n\t\tvar entryEnd = entry + (int)entrySize;\n\n\t\tif ( t.IsVideo )\n\t\t{\n\t\t\t// VisualSampleEntry: 6 reserved, data reference index, 16 bytes of pre-defined/reserved, then width and height\n\t\t\tt.Width = BinaryPrimitives.ReadUInt16BigEndian( d.AsSpan( entry + entryHead + 24 ) );\n\t\t\tt.Height = BinaryPrimitives.ReadUInt16BigEndian( d.AsSpan( entry + entryHead + 26 ) );\n\t\t\tvar configType = codec switch { \"avc1\" or \"avc3\" => \"avcC\", \"av01\" => \"av1C\", \"hvc1\" or \"hev1\" => \"hvcC\", _ => null };\n\t\t\tif ( configType is not null && Find( d, entry + entryHead + 78, entryEnd, configType ) is { } c )\n\t\t\t\tt.Config = d.AsSpan( c.at + c.h, c.size - c.h ).ToArray();\n\t\t}\n\t\telse if ( t.IsAudio )\n\t\t{\n\t\t\t// AudioSampleEntry: 6 reserved, data reference index, 8 reserved, channels, sample size, 4 reserved, rate 16.16\n\t\t\tt.Channels = BinaryPrimitives.ReadUInt16BigEndian( d.AsSpan( entry + entryHead + 16 ) );\n\t\t\tt.SampleRate = BinaryPrimitives.ReadUInt16BigEndian( d.AsSpan( entry + entryHead + 24 ) );\n\t\t\tif ( codec == \"mp4a\" && Find( d, entry + entryHead + 28, entryEnd, \"esds\" ) is { } esds )\n\t\t\t\tt.Config = AudioSpecificConfig( d.AsSpan( esds.at + esds.h + 4, esds.size - esds.h - 4 ) );\n\t\t}\n\t\treturn t;\n\t}\n\n\t/// <summary> The DecoderSpecificInfo (tag 5) inside an esds's descriptors: an AAC AudioSpecificConfig. </summary>\n\tstatic byte[] AudioSpecificConfig( ReadOnlySpan<byte> d )\n\t{\n\t\tvar i = 0;\n\t\twhile ( i + 2 <= d.Length )\n\t\t{\n\t\t\tvar tag = d[i++];\n\t\t\tvar length = 0;\n\t\t\tfor ( int k = 0; k < 4 && i < d.Length; k++ )\n\t\t\t{\n\t\t\t\tvar b = d[i++];\n\t\t\t\tlength = (length << 7) | (b & 0x7F);\n\t\t\t\tif ( (b & 0x80) == 0 ) break;\n\t\t\t}\n\t\t\tif ( tag == 5 ) return d.Slice( i, Math.Min( length, d.Length - i ) ).ToArray();\n\t\t\tif ( tag == 3 )\n\t\t\t{\n\t\t\t\t// ES_Descriptor: ES_ID, flags, then optional fields - its children follow\n\t\t\t\tvar flags = d[i + 2];\n\t\t\t\ti += 3;\n\t\t\t\tif ( (flags & 0x80) != 0 ) i += 2;\n\t\t\t\tif ( (flags & 0x40) != 0 ) i += 1 + d[i];\n\t\t\t\tif ( (flags & 0x20) != 0 ) i += 2;\n\t\t\t}\n\t\t\telse if ( tag == 4 ) i += 13; // DecoderConfigDescriptor: object type .. average bitrate, then children\n\t\t\telse i += length;\n\t\t}\n\t\treturn null;\n\t}\n\n\t/// <summary>\n\t/// The samples of every fragment (moof + mdat) in d[start..end], in order. <paramref name=\"fileOffset\"/> is where\n\t/// d[0] is in the file (for base data offsets); offsets in the result are into d. <paramref name=\"partial\"/>: the\n\t/// last fragment may be cut off - its samples stop where the data does.\n\t/// </summary>\n\tpublic static List<Sample> ReadSamples( byte[] d, int start, int end, IReadOnlyList<Track> tracks, long fileOffset = 0, bool partial = false )\n\t{\n\t\tvar samples = new List<Sample>();\n\t\tforeach ( var (moof, moofSize, type, moofHead) in Children( d, start, end ) )\n\t\t{\n\t\t\tif ( type != \"moof\" ) continue;\n\t\t\tforeach ( var (traf, trafSize, trafType, trafHead) in Children( d, moof + moofHead, moof + moofSize ) )\n\t\t\t{\n\t\t\t\tif ( trafType != \"traf\" ) continue;\n\t\t\t\tlong baseOffset = moof, decodeTime = 0;\n\t\t\t\tuint trackId = 0, duration = 0, size = 0, flags = 0;\n\t\t\t\tTrack track = null;\n\n\t\t\t\tforeach ( var (box, _, boxType, boxHead) in Children( d, traf + trafHead, traf + trafSize ) )\n\t\t\t\t{\n\t\t\t\t\tvar p = box + boxHead;\n\t\t\t\t\tif ( boxType == \"tfhd\" )\n\t\t\t\t\t{\n\t\t\t\t\t\tvar f = BinaryPrimitives.ReadUInt32BigEndian( d.AsSpan( p ) ) & 0xFFFFFF;\n\t\t\t\t\t\ttrackId = BinaryPrimitives.ReadUInt32BigEndian( d.AsSpan( p + 4 ) );\n\t\t\t\t\t\ttrack = tracks.FirstOrDefault( t => t.Id == trackId ) ?? (tracks.Count == 1 ? tracks[0] : null);\n\t\t\t\t\t\tduration = track?.DefaultDuration ?? 0;\n\t\t\t\t\t\tsize = track?.DefaultSize ?? 0;\n\t\t\t\t\t\tflags = track?.DefaultFlags ?? 0;\n\t\t\t\t\t\tp += 8; // version/flags, track_ID\n\t\t\t\t\t\tif ( (f & 0x01) != 0 ) { baseOffset = (long)BinaryPrimitives.ReadUInt64BigEndian( d.AsSpan( p ) ) - fileOffset; p += 8; }\n\t\t\t\t\t\tif ( (f & 0x02) != 0 ) p += 4;\n\t\t\t\t\t\tif ( (f & 0x08) != 0 ) { duration = BinaryPrimitives.ReadUInt32BigEndian( d.AsSpan( p ) ); p += 4; }\n\t\t\t\t\t\tif ( (f & 0x10) != 0 ) { size = BinaryPrimitives.ReadUInt32BigEndian( d.AsSpan( p ) ); p += 4; }\n\t\t\t\t\t\tif ( (f & 0x20) != 0 ) flags = BinaryPrimitives.ReadUInt32BigEndian( d.AsSpan( p ) );\n\t\t\t\t\t}\n\t\t\t\t\telse if ( boxType == \"tfdt\" )\n\t\t\t\t\t{\n\t\t\t\t\t\tdecodeTime = d[p] == 1 ? (long)BinaryPrimitives.ReadUInt64BigEndian( d.AsSpan( p + 4 ) ) : BinaryPrimitives.ReadUInt32BigEndian( d.AsSpan( p + 4 ) );\n\t\t\t\t\t}\n\t\t\t\t\telse if ( boxType == \"trun\" )\n\t\t\t\t\t{\n\t\t\t\t\t\tvar version = d[p];\n\t\t\t\t\t\tvar f = BinaryPrimitives.ReadUInt32BigEndian( d.AsSpan( p ) ) & 0xFFFFFF;\n\t\t\t\t\t\tvar count = BinaryPrimitives.ReadUInt32BigEndian( d.AsSpan( p + 4 ) );\n\t\t\t\t\t\tp += 8;\n\t\t\t\t\t\tlong data = baseOffset;\n\t\t\t\t\t\tif ( (f & 0x01) != 0 ) { data = baseOffset + BinaryPrimitives.ReadInt32BigEndian( d.AsSpan( p ) ); p += 4; }\n\t\t\t\t\t\tuint? firstFlags = null;\n\t\t\t\t\t\tif ( (f & 0x04) != 0 ) { firstFlags = BinaryPrimitives.ReadUInt32BigEndian( d.AsSpan( p ) ); p += 4; }\n\n\t\t\t\t\t\tfor ( uint i = 0; i < count; i++ )\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tuint sDuration = duration, sSize = size, sFlags = i == 0 && firstFlags is { } ff ? ff : flags;\n\t\t\t\t\t\t\tvar offset = 0;\n\t\t\t\t\t\t\tif ( (f & 0x100) != 0 ) { sDuration = BinaryPrimitives.ReadUInt32BigEndian( d.AsSpan( p ) ); p += 4; }\n\t\t\t\t\t\t\tif ( (f & 0x200) != 0 ) { sSize = BinaryPrimitives.ReadUInt32BigEndian( d.AsSpan( p ) ); p += 4; }\n\t\t\t\t\t\t\tif ( (f & 0x400) != 0 ) { sFlags = BinaryPrimitives.ReadUInt32BigEndian( d.AsSpan( p ) ); p += 4; }\n\t\t\t\t\t\t\tif ( (f & 0x800) != 0 ) { offset = BinaryPrimitives.ReadInt32BigEndian( d.AsSpan( p ) ); if ( version == 0 ) offset = (int)Math.Min( (uint)offset, int.MaxValue ); p += 4; }\n\n\t\t\t\t\t\t\tif ( partial && data >= 0 && data + sSize > Math.Min( end, d.Length ) ) return samples;\n\t\t\t\t\t\t\tif ( data < 0 || data + sSize > d.Length ) throw new MergeException( \"mp4 sample outside the fragment\" );\n\t\t\t\t\t\t\t// audio samples are all sync samples, whatever the flags say\n\t\t\t\t\t\t\tvar key = (sFlags & 0x10000) == 0 || (track?.IsAudio ?? false);\n\t\t\t\t\t\t\tsamples.Add( new Sample( track?.Id ?? trackId, decodeTime, offset, sDuration, key, (int)data, (int)sSize ) );\n\n\t\t\t\t\t\t\tdata += sSize;\n\t\t\t\t\t\t\tdecodeTime += sDuration;\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn samples;\n\t}\n\n\tpublic static (long size, string type, int headerLength) Box( byte[] d, int at, long remaining )\n\t{\n\t\tif ( at + 8 > d.Length ) return (0, null, 0);\n\t\tlong size = BinaryPrimitives.ReadUInt32BigEndian( d.AsSpan( at ) );\n\t\tvar type = Encoding.ASCII.GetString( d, at + 4, 4 );\n\t\tvar h = 8;\n\t\tif ( size == 1 )\n\t\t{\n\t\t\tif ( at + 16 > d.Length ) return (0, null, 0);\n\t\t\tsize = (long)BinaryPrimitives.ReadUInt64BigEndian( d.AsSpan( at + 8 ) );\n\t\t\th = 16;\n\t\t}\n\t\telse if ( size == 0 ) size = remaining;\n\t\treturn (size < h ? 0 : size, type, h);\n\t}\n\n\tpublic static IEnumerable<(int at, int size, string type, int head)> Children( byte[] d, int start, int end )\n\t{\n\t\tfor ( var at = start; at + 8 <= end; )\n\t\t{\n\t\t\tvar (size, type, h) = Box( d, at, end - at );\n\t\t\tif ( size <= 0 || at + size > end ) yield break;\n\t\t\tyield return (at, (int)size, type, h);\n\t\t\tat += (int)size;\n\t\t}\n\t}\n\n\tpublic static (int at, int size, int h)? Find( byte[] d, int start, int end, string type )\n\t{\n\t\tforeach ( var c in Children( d, start, end ) )\n\t\t\tif ( c.type == type ) return (c.at, c.size, c.head);\n\t\treturn null;\n\t}\n\n\t/// <summary> A box nested under a box at <paramref name=\"at\"/>, by the types of its ancestors below it. </summary>\n\tpublic static (int at, int size, int h)? Path( byte[] d, int at, int end, params string[] types )\n\t{\n\t\tvar head = Box( d, at, end - at ).headerLength;\n\t\t(int at, int size, int h)? found = null;\n\t\tint s = at + head, e = end;\n\t\tforeach ( var t in types )\n\t\t{\n\t\t\tfound = Find( d, s, e, t );\n\t\t\tif ( found is not { } f ) return null;\n\t\t\ts = f.at + f.h;\n\t\t\te = f.at + f.size;\n\t\t}\n\t\treturn found;\n\t}\n}\n"
},
{
"Ident": "rexi.bimp",
"Path": "Resolver/MediaInfo.cs",
"FileName": "MediaInfo.cs",
"PackageType": "library",
"CodeKind": "Game",
"AssetVersionId": 399292,
"IsPrivate": false,
"Code": "using System.Threading;\n\nnamespace Bimp.Resolver;\n\n/// <summary>\n/// A user facing resolve failure (\"This video is private.\").\n/// </summary>\npublic class ResolveException : Exception\n{\n\tpublic ResolveException( string message ) : base( message ) { }\n}\n\n/// <summary>\n/// What the host learns about a piece of media - shared with everyone through the synced queue item.\n/// Contains no stream urls: those are tied to the IP that asked for them, so every client gets its own.\n/// </summary>\npublic sealed class MediaInfo\n{\n\t/// <summary> The extractor that understood the url (<see cref=\"IExtractor.Key\"/>). </summary>\n\tpublic string Extractor { get; set; }\n\n\t/// <summary> Extractor specific id (YouTube video id, SoundCloud url...). </summary>\n\tpublic string Id { get; set; }\n\n\tpublic string Title { get; set; }\n\n\t/// <summary> Seconds, 0 if unknown. </summary>\n\tpublic float Duration { get; set; }\n\n\tpublic bool AudioOnly { get; set; }\n\tpublic string Thumbnail { get; set; }\n\n\t/// <summary> The stream is merged into local files by each client - it seeks by reopening at a time. </summary>\n\tpublic bool Merged { get; set; }\n\n\t/// <summary> A live stream (a Twitch channel...): played near its live edge, no seeking, no duration. </summary>\n\tpublic bool IsLive { get; set; }\n\n\t/// <summary> Video heights clients can pick, highest first. </summary>\n\tpublic List<int> Qualities { get; set; } = new();\n\n\t/// <summary> Audio tracks (dubs) clients can pick. Empty when there's no choice. </summary>\n\tpublic List<MediaAudioTrack> AudioTracks { get; set; } = new();\n\n\t/// <summary> Where to start playing (seconds), from the link (YouTube's <c>t=</c>). 0 = the beginning. </summary>\n\tpublic float StartAt { get; set; }\n\n\t/// <summary> Caption tracks clients can show. Empty when there are none. </summary>\n\tpublic List<MediaCaptionTrack> CaptionTracks { get; set; } = new();\n}\n\n/// <summary>\n/// What a client wants to play.\n/// </summary>\npublic sealed class StreamRequest\n{\n\tpublic bool AudioOnly { get; set; }\n\n\t/// <summary> The server's bimp_max_height. </summary>\n\tpublic int MaxHeight { get; set; } = 720;\n\n\t/// <summary> An exact height the client picked, 0 for automatic. </summary>\n\tpublic int Height { get; set; }\n\n\t/// <summary> Audio language the client picked, null for the original. </summary>\n\tpublic string Language { get; set; }\n\n\t/// <summary> A resolver client whose urls refused mid-download: re-resolving skips it, its fresh urls pass the probe and die the same way (see <see cref=\"StreamFile.ClientKey\"/>). </summary>\n\tpublic string AvoidClient { get; set; }\n\n\t/// <summary> A copy of this request that avoids a client whose urls just refused. </summary>\n\tpublic StreamRequest Avoiding( string clientKey ) => new() { AudioOnly = AudioOnly, MaxHeight = MaxHeight, Height = Height, Language = Language, AvoidClient = clientKey };\n}\n\npublic enum StreamKind\n{\n\t/// <summary> A url the engine plays as-is (progressive mp4/webm/mp3). </summary>\n\tDirect,\n\n\t/// <summary> Separate video + audio WebM files, merged locally into segment files. </summary>\n\tMerge,\n\n\t/// <summary> A single audio WebM, re-laid out locally (cues at the end) for audio only playback. </summary>\n\tAudio,\n\n\t/// <summary> A finished HLS playlist (<see cref=\"StreamPlan.DirectUrl\"/>), remuxed locally from a start time. </summary>\n\tHls,\n\n\t/// <summary> A live stream url (<see cref=\"StreamPlan.DirectUrl\"/>: HLS, MPEG-TS, RTSP), played like a pasted live link. </summary>\n\tLive,\n}\n\n/// <summary>\n/// One downloadable file of a stream.\n/// </summary>\npublic sealed class StreamFile\n{\n\tpublic string Url { get; set; }\n\n\t/// <summary> Bytes, 0 if unknown. </summary>\n\tpublic long Size { get; set; }\n\n\t/// <summary> Extractor specific id of the format (YouTube itag), to find the same file again after a refresh. </summary>\n\tpublic string FormatId { get; set; }\n\n\t/// <summary> \"webm\", or \"mp4\" for a fragmented MP4 (YouTube's AV1). </summary>\n\tpublic string Container { get; set; } = \"webm\";\n\n\t/// <summary> The resolver client that minted Url (null off YouTube). Media GETs identify as it, and a refusal re-resolves past it - a dead client's fresh urls pass the probe all the same. </summary>\n\tpublic string ClientKey { get; set; }\n}\n\n/// <summary>\n/// How a client plays a piece of media, resolved on that client.\n/// </summary>\npublic sealed class StreamPlan\n{\n\tpublic StreamKind Kind { get; set; }\n\tpublic StreamFile Video { get; set; }\n\tpublic StreamFile Audio { get; set; }\n\n\t/// <summary> A video to use instead if <see cref=\"Video\"/> can't be read (the VP9 copy of an AV1 choice). </summary>\n\tpublic StreamFile FallbackVideo { get; set; }\n\n\t/// <summary>\n\t/// A video to carry on with once <see cref=\"Video\"/> runs out: YouTube serves its AV1 4K60 to the clients that get it\n\t/// for only ~88 MB (about a minute), so playback switches to VP9 1440p60 at a keyframe (see WebmSegmenter).\n\t/// </summary>\n\tpublic StreamFile SwitchVideo { get; set; }\n\n\t/// <summary> For <see cref=\"StreamKind.Direct\"/>, <see cref=\"StreamKind.Hls\"/> and <see cref=\"StreamKind.Live\"/>. </summary>\n\tpublic string DirectUrl { get; set; }\n\n\t/// <summary> The client that minted the plan's urls - an AV1 <see cref=\"Video\"/> can come from another one (see <see cref=\"StreamFile.ClientKey\"/>). </summary>\n\tpublic string ClientKey { get; set; }\n}\n\n/// <summary>\n/// Understands one site. Stateless apart from caches, safe to share.\n/// </summary>\npublic interface IExtractor\n{\n\t/// <summary> Short name, used in play tokens (\"yt\"). </summary>\n\tstring Key { get; }\n\n\tbool CanHandle( Uri url );\n\n\t/// <summary> Host: title, duration, choices. Throws <see cref=\"ResolveException\"/>. </summary>\n\tTask<MediaInfo> GetInfoAsync( Uri url, bool audioOnly, CancellationToken ct );\n\n\t/// <summary> Client: the files to play. Throws <see cref=\"ResolveException\"/>. </summary>\n\tTask<StreamPlan> GetStreamsAsync( string id, StreamRequest request, CancellationToken ct );\n}\n\n/// <summary> An extractor whose media can have captions: <see cref=\"MediaInfo.CaptionTracks\"/> lists them, this fetches one. </summary>\npublic interface ICaptionExtractor\n{\n\t/// <summary>\n\t/// Client: the raw caption file of one track (<see cref=\"MediaCaptionTrack.Id\"/>), for <see cref=\"Captions.Parse\"/>.\n\t/// Fetched per client because the urls can be tied to the requesting IP. Null when it can't be had.\n\t/// </summary>\n\tTask<string> GetCaptionsAsync( string id, string trackId, CancellationToken ct );\n}\n\n/// <summary> One video of a playlist. </summary>\npublic sealed record PlaylistEntry( string Url, string Title, float Duration );\n\n/// <summary> An extractor that can list the videos of a playlist link. </summary>\npublic interface IPlaylistExtractor\n{\n\t/// <summary> The entries (at most <paramref name=\"max\"/>) if this url is a playlist without a single video picked, else null. </summary>\n\tTask<List<PlaylistEntry>> GetPlaylistAsync( Uri url, int max, CancellationToken ct );\n}\n"
},
{
"Ident": "rexi.bimp",
"Path": "SegmentPlayer.cs",
"FileName": "SegmentPlayer.cs",
"PackageType": "library",
"CodeKind": "Game",
"AssetVersionId": 399292,
"IsPrivate": false,
"Code": "using Bimp.Resolver.Media;\n\nnamespace Bimp;\n\n/// <summary>\n/// Plays a series of local segment files back to back: merged YouTube media (<see cref=\"WebmSegmenter\"/>) or a\n/// live stream remuxed to MP4 (<see cref=\"Resolver.Live.LiveSegmenter\"/>).\n/// <para>\n/// Segment N plays while N+1 is started just in time in a second, hidden and silent player; at the boundary\n/// they swap, so the picture carries straight on. Segments keep absolute timestamps, so <see cref=\"Time\"/>\n/// is the media time.\n/// </para>\n/// </summary>\npublic sealed class SegmentPlayer : IDisposable\n{\n\tclass Slot\n\t{\n\t\tpublic MediaSegment Segment;\n\t\tpublic VideoPlayer Player;\n\t\tpublic bool Loaded;\n\t\tpublic bool Finished;\n\t\tpublic bool Preload; // waiting in the wings: silent until it's swapped in\n\t\tpublic bool FirstFrame; // set when it delivers its first frame (from OnTextureData, during Present)\n\t\tpublic bool Primed; // its decoder has produced its first frame\n\t\tpublic int Steady; // frames in a row it has delivered on time (see SettleFrames)\n\t\tpublic RealTimeSince SinceFrame;\n\t\tpublic bool Fading; // swapped out: silent (its volume ramps down) and closed a moment later\n\t\tpublic float LastTime;\n\t\tpublic RealTimeSince SinceOpened;\n\t}\n\n\treadonly ISegmentFeed feed;\n\n\t/// <summary>\n\t/// Live: start this many segments behind the newest finished one, so the next is always ready by the time the\n\t/// current one ends. Latency is roughly (JoinBehind + 1) segments plus the overlap.\n\t/// </summary>\n\tint JoinBehind => feed.LowLatency ? 0 : 1;\n\n\t/// <summary> Live: further behind than this (slow machine, stalled download) jumps back to the live edge. </summary>\n\tint MaxBehind => feed.LowLatency ? 1 : 3;\n\n\t/// <summary>\n\t/// Low latency: join the newest segment this long after it's written, so the next one is ready a little before\n\t/// the current one ends (its start latency plus arrival jitter). A segment that's late anyway just holds the\n\t/// last picture (the filler frames) until it arrives.\n\t/// </summary>\n\tconst double LowJoinMargin = 0.2;\n\n\t/// <summary>\n\t/// Average time from opening a segment to its first decoded frame (measured as we go).\n\t/// </summary>\n\tfloat startLatency;\n\n\t/// <summary>\n\t/// How long before the current segment ends to start the next one, so its first frame lands on the boundary -\n\t/// and so the two players are close to in step for the audio crossfade. A player that isn't primed in time\n\t/// just swaps a moment later (the current one plays on into its overlap, if the file has one).\n\t/// </summary>\n\tfloat StartLead => JustInTime\n\t\t? Math.Clamp( (startLatency > 0 ? startLatency : 0.08f) + 0.015f, 0.0f, 1.0f )\n\t\t: ClockLeadOnly ? Math.Clamp( clockLead, 0.0f, Math.Min( current.Segment.Overlap, 1.0f ) )\n\t\t: Math.Clamp( (startLatency > 0 ? startLatency : 0.04f) + leadCorrection, 0.0f, feed.IsLive ? 1.0f : 3.0f );\n\n\t/// <summary>\n\t/// Media files that run on into the next segment (<see cref=\"MediaSegment.Overlap\"/>): the current player shows real\n\t/// frames until the next one's first frame is up, however long that takes (150-520 ms at 1440p/4K), so the next one\n\t/// only has to be opened for its clock to be in step. Adding the start latency to that made it skip that far ahead\n\t/// at a swap, and correcting that back made the current one hold its last frame instead.\n\t/// <para>\n\t/// Merged WebM (VP9/AV1) only: MP4 segments (HLS VOD) kept waiting for the next H.264 player to settle made Media\n\t/// Foundation drop a sample (\"MFT deadlock\") in every run, and the picture broke up until the next keyframe (3 of 3\n\t/// runs, against 0 of 3 with the old timing).\n\t/// </para>\n\t/// </summary>\n\tbool ClockLeadOnly => feed is WebmSegmenter && (current?.Segment.Overlap ?? 0) > 0;\n\n\t/// <summary> How long before the end to open the next player so its clock is in step (<see cref=\"ClockLeadOnly\"/>), learned from each swap. </summary>\n\tfloat clockLead = 0.05f;\n\n\t/// <summary>\n\t/// <see cref=\"ClockLeadOnly\"/>: swap once the next player has delivered this many frames in a row on time. Its first\n\t/// frame comes late, and a 1440p decoder is still catching up to its clock for a moment after it - swapping on the\n\t/// first frame put that on screen as a 80-120 ms gap (measured). The current one plays on into its overlap meanwhile.\n\t/// </summary>\n\tconst int SettleFrames = 4;\n\tconst float SettleInterval = 0.045f;\n\n\t/// <summary>\n\t/// Low latency files that end at their boundary (H.264, ending in skip frames): the next one just opens early enough\n\t/// for its first frame to be ready there. (Syncing clocks made every swap ~40 ms late: a new player's clock runs from\n\t/// Play(), its first frame comes ~60 ms later.)\n\t/// </summary>\n\tbool JustInTime => feed.LowLatency && (current?.Segment.Overlap ?? 0) <= 0;\n\n\t/// <summary>\n\t/// Learned from each swap: how far the next player's clock was ahead of the current one's at the boundary.\n\t/// Starting it that much later next time brings them in step, which the audio crossfade needs - and so the next\n\t/// player neither skips its first frames nor makes the current one hold its last, while the current one plays on\n\t/// into its overlap (see <see cref=\"MediaSegment.Overlap\"/>).\n\t/// </summary>\n\tfloat leadCorrection;\n\n\t/// <summary>\n\t/// How long the swapped-out player keeps running, silent, after a swap.\n\t/// <para>\n\t/// The engine's mixer runs every sound through Steam Audio's direct effect, whose gain interpolator ramps any\n\t/// volume change over the next mix frames. Unmuting the next player while closing the current one outright\n\t/// left a ~70 ms dip in the audio at every boundary (measured with a sine: down to 5%, every segment).\n\t/// Muting the old player at the same moment instead - both ramps at once - makes that a crossfade.\n\t/// </para>\n\t/// </summary>\n\tconst float FadeSeconds = 0.15f;\n\n\tSlot fading;\n\tRealTimeSince sinceFade;\n\n\tSlot current, next;\n\tbool paused;\n\n\t// audio settings, applied to both players\n\tVector3 audioPosition;\n\tbool audioSpatial;\n\tfloat audioVolume = 1;\n\tfloat audioDistance = 2500;\n\n\tstring error;\n\tpublic string Error => error ?? feed.Error;\n\n\t/// <summary> The segment on screen, -1 before the first. </summary>\n\tpublic int CurrentSeq => current?.Segment.Seq ?? -1;\n\n\t/// <summary> Has the first segment started? </summary>\n\tpublic bool Loaded => current is { Loaded: true } || (Mjpeg?.HasShown ?? false);\n\n\t/// <summary> Motion JPEG: frames go straight to the texture, there are no segments to play. </summary>\n\tResolver.Live.LiveMjpeg Mjpeg => (feed as Resolver.Live.LiveSegmenter)?.Mjpeg;\n\n\t/// <summary> The last segment has played to its end. </summary>\n\tpublic bool Finished { get; private set; }\n\n\t/// <summary> Media time (VOD), or running stream time (live). </summary>\n\tpublic float Time => (float)StreamTime;\n\n\t/// <summary>\n\t/// Stream time (seconds) of what's on screen. WebM segments keep absolute timestamps (their StreamStart is 0); MP4\n\t/// segments (live, HLS VOD) each start their clock at 0.\n\t/// </summary>\n\tpublic double StreamTime => Mjpeg is { } mjpeg ? mjpeg.ShownTime : (current?.Segment.StreamStart ?? 0) + (current?.Player?.PlaybackTime ?? 0);\n\n\tpublic float Duration => current?.Player?.Duration ?? 0;\n\tpublic bool IsPaused => current?.Player?.IsPaused ?? paused;\n\n\t/// <summary> Every segment's frames go to one texture (see <see cref=\"VideoFrameSink\"/>). </summary>\n\treadonly VideoFrameSink sink = new();\n\n\tpublic Texture Texture => sink.Texture;\n\tpublic int Width => sink.Width;\n\tpublic int Height => sink.Height;\n\tpublic bool HasVideo => sink.Texture is not null && Width > 4 && Height > 4;\n\tpublic (int rate, int channels) AudioFormat => (current?.Player?.SampleRate ?? 0, current?.Player?.Channels ?? 0);\n\n\t/// <summary>\n\t/// Merged media: the synced media time, null while paused. A first segment that starts later than it (at the next\n\t/// keyframe - see <see cref=\"MediaBackend\"/>) opens only when the clock gets there, so it never has to seek forward.\n\t/// </summary>\n\tpublic float? SyncedTime { get; set; }\n\n\tpublic SegmentPlayer( ISegmentFeed feed )\n\t{\n\t\tthis.feed = feed;\n\t\tif ( !feed.IsLive ) feed.Want( 0 );\n\t}\n\n\t/// <summary>\n\t/// Call every frame.\n\t/// </summary>\n\tpublic void Update()\n\t{\n\t\tif ( Mjpeg is { } mjpeg )\n\t\t{\n\t\t\tif ( !paused ) mjpeg.Present( sink );\n\t\t\treturn;\n\t\t}\n\n\t\tif ( current is null )\n\t\t{\n\t\t\tvar first = feed.IsLive ? LiveJoinSegment() : feed.TryGet( 0 );\n\t\t\tif ( first is null ) return;\n\t\t\tif ( !feed.IsLive && !paused && SyncedTime is { } now && first.StreamStart + first.Start - now > StartLead ) return;\n\t\t\tcurrent = Open( first );\n\t\t\tif ( paused ) current.Player?.Pause();\n\t\t}\n\n\t\t// Live and fell far behind? Jump to the live edge.\n\t\tif ( feed.IsLive && feed.Newest - current.Segment.Seq > MaxBehind && LiveJoinSegment() is { } edge )\n\t\t{\n\t\t\tLog.Info( $\"[bimp] live: {feed.Newest - current.Segment.Seq} segments behind, jumping to the live edge\" );\n\t\t\tClose( next ); next = null;\n\t\t\tClose( current );\n\t\t\tcurrent = Open( edge );\n\t\t}\n\n\t\tTimedPresent( current, \"current\" );\n\n\t\t// the first segment's own start seeds the start latency, so the first swap isn't late on a slow (4K) decoder\n\t\tif ( startLatency <= 0 && !current.Primed && current.FirstFrame )\n\t\t{\n\t\t\tcurrent.Primed = true;\n\t\t\tstartLatency = (float)current.SinceOpened;\n\t\t}\n\n\t\t// the swapped-out player finishes its fade, then goes\n\t\tif ( fading is not null )\n\t\t{\n\t\t\tTimedPresent( fading, \"fading\" );\n\t\t\tif ( sinceFade > FadeSeconds ) CloseFading();\n\t\t}\n\n\t\tvar p = current.Player;\n\t\tvar seg = current.Segment;\n\n\t\t// A player started fresh delivers its first frame ~0.25s after Play(), and one that was paused takes ~1s\n\t\t// after Resume() - so start the next segment just in time (hidden, silent, presented so it runs),\n\t\t// timed from the measured start latency, and swap which player is on screen. Nothing ever pauses.\n\t\tif ( next is null && !paused && !seg.Last && current.Loaded && p is not null && p.PlaybackTime >= seg.End - StartLead )\n\t\t{\n\t\t\tvar upcoming = feed.TryGet( seg.Seq + 1 );\n\t\t\tif ( upcoming is not null ) next = Open( upcoming, preload: true );\n\t\t}\n\n\t\tif ( next?.Player is not null )\n\t\t{\n\t\t\tTimedPresent( next, \"next\" );\n\n\t\t\tif ( !next.Primed && next.FirstFrame )\n\t\t\t{\n\t\t\t\tnext.Primed = true;\n\t\t\t\tvar latency = (float)next.SinceOpened;\n\t\t\t\tstartLatency = startLatency <= 0 ? latency : startLatency * 0.7f + latency * 0.3f;\n\t\t\t\tMediaProbe.Note( $\"seg {next.Segment.Seq} first frame {latency * 1000:0}ms after opening (avg {startLatency * 1000:0}ms), current at {p?.PlaybackTime:0.000}/{seg.End:0.000}\" );\n\t\t\t}\n\t\t}\n\n\t\t// Swap when the current segment's clock reaches its end. (OnFinished fires ~80ms early, while\n\t\t// frames are still queued - swapping on it cut off the last frames.)\n\t\tvar stuck = current.Finished && p is not null && MathF.Abs( p.PlaybackTime - current.LastTime ) < 0.0001f;\n\t\tcurrent.LastTime = p?.PlaybackTime ?? 0;\n\t\tvar atEnd = (current.Loaded && p is not null && p.PlaybackTime >= seg.End - 0.005f) || stuck;\n\t\tif ( !atEnd ) return;\n\n\t\tif ( seg.Last )\n\t\t{\n\t\t\tFinished = current.Finished || stuck;\n\t\t\treturn;\n\t\t}\n\n\t\t// a next player still catching up waits, while the current one has overlap left to play\n\t\tvar settled = !ClockLeadOnly || next is null || next.Steady >= SettleFrames || (p is not null && p.PlaybackTime >= seg.End + seg.Overlap - 0.1f) || stuck;\n\t\tif ( next is { Primed: true } && settled )\n\t\t{\n\t\t\t// the next player's clock vs the current one's, in stream time: + means it's ahead. (Merged media keeps\n\t\t\t// absolute timestamps; live segments each start their clock at 0.)\n\t\t\tvar ahead = (float)((next.Segment.StreamStart + (next.Player?.PlaybackTime ?? 0)) - (seg.StreamStart + (p?.PlaybackTime ?? seg.End)));\n\n\t\t\t// keep the two clocks in step (see leadCorrection)\n\t\t\tif ( ClockLeadOnly ) clockLead = Math.Clamp( clockLead - ahead * 0.7f, 0.0f, 1.0f );\n\t\t\telse if ( !JustInTime ) leadCorrection = Math.Clamp( leadCorrection - ahead * 0.7f, -0.2f, 0.2f );\n\t\t\tMediaProbe.Note( $\"swap {seg.Seq}->{next.Segment.Seq} at {p?.PlaybackTime:0.000}/{seg.End:0.000} (overlap {seg.Overlap * 1000:0}ms), next ahead by {ahead * 1000:0}ms, lead now {StartLead * 1000:0}ms\" );\n\t\t\tnext.Preload = false;\n\t\t\tApplyAudio( next );\n\n\t\t\t// its frame from just before the boundary goes up now, not 33 ms later with its next one\n\t\t\tsink.ShowHeld( next.Player );\n\n\t\t\t// crossfade: the old player goes silent as the new one comes up (see FadeSeconds)\n\t\t\tCloseFading();\n\t\t\tfading = current;\n\t\t\tfading.Fading = true;\n\t\t\tApplyAudio( fading );\n\t\t\tsinceFade = 0;\n\n\t\t\tcurrent = next;\n\t\t\tnext = null;\n\t\t\tfeed.Want( current.Segment.Seq ); // keeps the one after it coming\n\t\t}\n\n\t\t// otherwise the next segment isn't downloaded yet - hold the last frame\n\t}\n\n\tvoid TimedPresent( Slot slot, string role )\n\t{\n\t\tif ( slot?.Player is null ) return;\n\t\tvar t = System.Diagnostics.Stopwatch.GetTimestamp();\n\t\tslot.Player.Present();\n\t\tMediaProbe.PresentTook( System.Diagnostics.Stopwatch.GetElapsedTime( t ).TotalMilliseconds, role, slot.Segment.Seq );\n\t}\n\n\t/// <summary>\n\t/// Live: the segment to start from - the newest one that's at least <see cref=\"JoinBehind\"/> behind the newest\n\t/// finished segment and leaves the feed's start buffer of stream time after it. Null while there isn't enough yet.\n\t/// </summary>\n\tMediaSegment LiveJoinSegment()\n\t{\n\t\tvar newest = feed.TryGet( feed.Newest );\n\t\tif ( newest is null ) return null;\n\t\tvar bufferedUntil = newest.StreamStart + newest.End;\n\n\t\tif ( feed.LowLatency ) return RealTime.NowDouble - newest.ReadyAt >= LowJoinMargin ? newest : null;\n\n\t\tfor ( var seq = newest.Seq - JoinBehind; seq >= 0; seq-- )\n\t\t{\n\t\t\tvar s = feed.TryGet( seq );\n\t\t\tif ( s is null ) return null; // not enough on disk yet\n\t\t\tif ( bufferedUntil - s.StreamStart >= feed.StartBufferSeconds ) return s;\n\t\t}\n\t\treturn null;\n\t}\n\n\tSlot Open( MediaSegment segment, bool preload = false )\n\t{\n\t\tMediaProbe.Note( $\"open seg {segment.Seq} ({segment.Start:0.00}-{segment.End:0.00}){(preload ? \" (preload)\" : \"\")}\" );\n\t\tvar slot = new Slot { Segment = segment, SinceOpened = 0, Preload = preload };\n\t\ttry\n\t\t{\n\t\t\tvar player = new VideoPlayer();\n\t\t\tplayer.OnLoaded += () => slot.Loaded = true;\n\t\t\tplayer.OnFinished += () => slot.Finished = true;\n\t\t\tslot.Player = player;\n\t\t\tApplyAudio( slot );\n\t\t\t// only the player on screen updates the texture; a preloading one just says it's ready\n\t\t\tsink.Attach( player, () => !slot.Preload && !slot.Fading, () =>\n\t\t\t{\n\t\t\t\tslot.Steady = slot.FirstFrame && slot.SinceFrame < SettleInterval ? slot.Steady + 1 : 0;\n\t\t\t\tslot.SinceFrame = 0;\n\t\t\t\tslot.FirstFrame = true;\n\t\t\t}, $\"seg {segment.Seq}\", holdWhileInactive: preload );\n\t\t\tvar playTimer = System.Diagnostics.Stopwatch.StartNew();\n\t\t\tplayer.Play( FileSystem.Data, segment.Path );\n\t\t\tMediaProbe.Note( $\"Play() seg {segment.Seq} took {playTimer.Elapsed.TotalMilliseconds:0.0}ms\" );\n\t\t}\n\t\tcatch ( Exception e )\n\t\t{\n\t\t\terror = e.Message;\n\t\t}\n\t\treturn slot;\n\t}\n\n\tvoid Close( Slot slot )\n\t{\n\t\tif ( slot?.Player is null ) return;\n\t\tvar timer = System.Diagnostics.Stopwatch.StartNew();\n\t\tsink.Detach( slot.Player );\n\t\tslot.Player.Dispose();\n\t\tMediaProbe.Note( $\"close seg {slot.Segment.Seq} took {timer.Elapsed.TotalMilliseconds:0.0}ms\" );\n\t}\n\n\tvoid CloseFading()\n\t{\n\t\tif ( fading is null ) return;\n\t\tClose( fading );\n\t\tfeed.Release( fading.Segment.Seq );\n\t\tfading = null;\n\t}\n\n\tpublic void SetPaused( bool pause )\n\t{\n\t\tpaused = pause;\n\t\tif ( current?.Player is { } p )\n\t\t{\n\t\t\tif ( pause ) p.Pause(); else p.Resume();\n\t\t}\n\n\t\tif ( pause ) CloseFading();\n\n\t\t// a preloading player would carry on running - start it again when we resume\n\t\tif ( pause && next is not null )\n\t\t{\n\t\t\tClose( next );\n\t\t\tnext = null;\n\t\t}\n\t}\n\n\t/// <summary>\n\t/// A short hop within the current segment (landing exactly on the synced time after opening).\n\t/// Anything further is done by opening a new segmenter at the time.\n\t/// </summary>\n\tpublic void Seek( float time )\n\t{\n\t\tvar seg = current?.Segment;\n\t\tif ( seg is null ) return;\n\t\tcurrent.Player?.Seek( Math.Clamp( time - (float)seg.StreamStart, seg.Start, seg.End ) );\n\t\tFinished = false;\n\t}\n\n\tpublic void SetAudio( Vector3 position, bool spatial, float volume, float distance )\n\t{\n\t\taudioPosition = position;\n\t\taudioSpatial = spatial;\n\t\taudioVolume = volume;\n\t\taudioDistance = distance;\n\t\tApplyAudio( current );\n\t\tApplyAudio( next );\n\t}\n\n\tvoid ApplyAudio( Slot slot )\n\t{\n\t\tif ( slot?.Player is null ) return;\n\t\tvar a = slot.Player.Audio;\n\t\ta.ListenLocal = !audioSpatial;\n\t\ta.Position = audioSpatial ? audioPosition : Vector3.Forward * 64.0f;\n\t\ta.Volume = slot.Preload || slot.Fading ? 0 : audioVolume;\n\t\ta.Distance = audioDistance;\n\t\ta.Falloff = MediaPlayer.AudioFalloff;\n\t}\n\n\tpublic void Dispose()\n\t{\n\t\tCloseFading();\n\t\tClose( current );\n\t\tClose( next );\n\t\tcurrent = next = null;\n\t\tsink.Dispose();\n\t}\n}\n"
},
{
"Ident": "rexi.bimp",
"Path": "UseHandled.cs",
"FileName": "UseHandled.cs",
"PackageType": "library",
"CodeKind": "Game",
"AssetVersionId": 399292,
"IsPrivate": false,
"Code": "namespace Bimp;\n\n/// <summary>\n/// One USE key press can reach us several ways in the same frame: the screen's own look-and-press\n/// (<see cref=\"MediaScreen\"/>), the PlayerController pressing <see cref=\"MediaInteract\"/> when in reach,\n/// and the remote's close-on-use. Whoever runs first handles it, the rest see it's taken.\n/// </summary>\ninternal static class UseHandled\n{\n\tstatic float handledAt = -1;\n\n\t/// <summary>\n\t/// Returns true if nobody has handled a USE press this frame (and marks it handled).\n\t/// </summary>\n\tpublic static bool TryConsume()\n\t{\n\t\tif ( handledAt == Time.Now ) return false;\n\t\thandledAt = Time.Now;\n\t\treturn true;\n\t}\n}\n"
},
{
"Ident": "rexi.bimp",
"Path": "MediaRemote.razor",
"FileName": "MediaRemote.razor",
"PackageType": "library",
"CodeKind": "Game",
"AssetVersionId": 399292,
"IsPrivate": false,
"Code": "@using System\n@using System.Linq\n@using Sandbox\n@using Sandbox.UI\n@namespace Bimp\n@inherits PanelComponent\n@implements Component.ISceneStage\n\n@*\n\tThe full controls, shown on the local player's screen when they USE a media player.\n\tCreated on demand by MediaRemote.Open, never networked.\n*@\n\n<root class=\"mediaremote @(PadMode ? \"pad\" : \"\")\">\n\n\t@if ( Target is not null )\n\t{\n\t\t<div class=\"window\">\n\n\t\t\t<div class=\"header\">\n\t\t\t\t<i>@(Target.AudioOnlyPlayer ? \"speaker\" : \"smart_display\")</i>\n\t\t\t\t<div class=\"title\">Media Player</div>\n\t\t\t\t<div class=\"close nav\" onclick=\"@Close\"><i>close</i></div>\n\t\t\t</div>\n\n\t\t\t<div class=\"now\">\n\t\t\t\t@if ( Target.HasMedia )\n\t\t\t\t{\n\t\t\t\t\t<div class=\"label\">Now playing</div>\n\t\t\t\t\t<div class=\"track\">@Target.Title</div>\n\t\t\t\t\t<div class=\"sub\">\n\t\t\t\t\t\t@if ( !string.IsNullOrWhiteSpace( Target.RequestedBy ) )\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t<span>Requested by @Target.RequestedBy</span>\n\t\t\t\t\t\t}\n\t\t\t\t\t\t<span class=\"time\">@TimeText</span>\n\t\t\t\t\t</div>\n\t\t\t\t}\n\t\t\t\telse\n\t\t\t\t{\n\t\t\t\t\t<div class=\"track dim\">Nothing playing</div>\n\t\t\t\t}\n\n\t\t\t\t@if ( !string.IsNullOrWhiteSpace( Target.Status ) )\n\t\t\t\t{\n\t\t\t\t\t<div class=\"status\">@Target.Status</div>\n\t\t\t\t}\n\t\t\t\t@if ( !string.IsNullOrWhiteSpace( Target.LocalNotice ) )\n\t\t\t\t{\n\t\t\t\t\t<div class=\"notice\">@Target.LocalNotice</div>\n\t\t\t\t}\n\t\t\t</div>\n\n\t\t\t@if ( Target.HasMedia && !Target.IsLive && Target.MediaDuration > 0 )\n\t\t\t{\n\t\t\t\t<div class=\"seek @(Target.CanLocalControl ? \"nav\" : \"disabled\")\">\n\t\t\t\t\t<SliderControl Min=\"@(0f)\" Max=\"@Target.MediaDuration\" Step=\"@(1f)\" ShowValueTooltip=\"@false\" Value=\"@SeekValue\" OnValueChanged=\"@OnSeekChanged\"></SliderControl>\n\t\t\t\t</div>\n\t\t\t}\n\n\t\t\t@if ( Target.CanLocalControl && Target.HasMedia )\n\t\t\t{\n\t\t\t\t<div class=\"transport\">\n\t\t\t\t\t@if ( !Target.IsLive )\n\t\t\t\t\t{\n\t\t\t\t\t\t<div class=\"btn nav\" onclick=\"@( () => Target.RequestSeek( Target.CurrentTime - 10 ) )\"><i>replay_10</i></div>\n\t\t\t\t\t\t<div class=\"btn big nav nav-default\" onclick=\"@( () => Target.RequestSetPaused( !Target.Paused ) )\"><i>@(Target.Paused ? \"play_arrow\" : \"pause\")</i></div>\n\t\t\t\t\t\t<div class=\"btn nav\" onclick=\"@( () => Target.RequestSeek( Target.CurrentTime + 10 ) )\"><i>forward_10</i></div>\n\t\t\t\t\t}\n\t\t\t\t\t<div class=\"btn nav\" onclick=\"@Target.RequestSkip\"><i>skip_next</i></div>\n\t\t\t\t\t<div class=\"btn nav\" onclick=\"@Target.RequestStop\"><i>stop</i></div>\n\t\t\t\t</div>\n\t\t\t}\n\n\t\t\t@if ( Target.HasMedia && (Target.AvailableQualities.Count > 0 || Target.AvailableAudioTracks.Count > 0 || Target.AvailableCaptionTracks.Count > 0) )\n\t\t\t{\n\t\t\t\t<div class=\"stream-options\">\n\t\t\t\t\t@if ( Target.AvailableQualities.Count > 0 )\n\t\t\t\t\t{\n\t\t\t\t\t\t<div class=\"option\">\n\t\t\t\t\t\t\t<i>high_quality</i>\n\t\t\t\t\t\t\t<DropDown class=\"quality nav\" Value=\"@QualityValue\" ValueChanged=\"@OnQualityChanged\" BuildOptions=\"@BuildQualityOptions\"></DropDown>\n\t\t\t\t\t\t</div>\n\t\t\t\t\t}\n\t\t\t\t\t@if ( Target.AvailableAudioTracks.Count > 0 )\n\t\t\t\t\t{\n\t\t\t\t\t\t<div class=\"option\">\n\t\t\t\t\t\t\t<i>translate</i>\n\t\t\t\t\t\t\t<DropDown class=\"audio-track nav\" Value=\"@AudioTrackValue\" ValueChanged=\"@OnAudioTrackChanged\" BuildOptions=\"@BuildAudioTrackOptions\"></DropDown>\n\t\t\t\t\t\t</div>\n\t\t\t\t\t}\n\t\t\t\t\t@if ( Target.AvailableCaptionTracks.Count > 0 )\n\t\t\t\t\t{\n\t\t\t\t\t\t<div class=\"option\">\n\t\t\t\t\t\t\t<i>closed_caption</i>\n\t\t\t\t\t\t\t<DropDown class=\"audio-track nav\" Value=\"@CaptionValue\" ValueChanged=\"@OnCaptionChanged\" BuildOptions=\"@BuildCaptionOptions\"></DropDown>\n\t\t\t\t\t\t</div>\n\t\t\t\t\t}\n\t\t\t\t\t<div class=\"playing-as\">@PlayingAsText</div>\n\t\t\t\t</div>\n\t\t\t}\n\n\t\t\t@if ( (Target.CanLocalControl || Target.CanLocalQueue) && PadMode )\n\t\t\t{\n\t\t\t\t@* a pad can't type a link *@\n\t\t\t\t<div class=\"pad-note\"><i>keyboard</i> Use a keyboard to add a link</div>\n\t\t\t}\n\t\t\telse if ( Target.CanLocalControl || Target.CanLocalQueue )\n\t\t\t{\n\t\t\t\t<div class=\"entry\">\n\t\t\t\t\t<TextEntry @ref=\"UrlEntry\" Placeholder=\"Paste a YouTube link or a media url (mp4, webm, mp3...)\" onsubmit=\"@OnSubmit\"></TextEntry>\n\t\t\t\t\t@if ( Target.CanLocalControl )\n\t\t\t\t\t{\n\t\t\t\t\t\t<div class=\"action primary\" onclick=\"@PlayNow\"><i>play_arrow</i> Play now</div>\n\t\t\t\t\t}\n\t\t\t\t\t@if ( Target.CanLocalQueue )\n\t\t\t\t\t{\n\t\t\t\t\t\t<div class=\"action\" onclick=\"@AddToQueue\"><i>playlist_add</i> Queue</div>\n\t\t\t\t\t}\n\t\t\t\t</div>\n\t\t\t}\n\t\t\telse\n\t\t\t{\n\t\t\t\t<div class=\"locked\"><i>lock</i> You can't control this player.</div>\n\t\t\t}\n\n\t\t\t<div class=\"queue\">\n\t\t\t\t<div class=\"queue-header\">\n\t\t\t\t\t<div class=\"tab nav @(ShowRecent ? \"\" : \"active\")\" onclick=\"@( () => ShowRecent = false )\">Up next <span class=\"count\">@Target.Queue.Count / @Target.MaxQueue</span></div>\n\t\t\t\t\t<div class=\"tab nav @(ShowRecent ? \"active\" : \"\")\" onclick=\"@( () => ShowRecent = true )\">Recent <span class=\"count\">@RecentLinks.All.Count</span></div>\n\t\t\t\t\t<div class=\"spacer\"></div>\n\t\t\t\t\t@if ( !ShowRecent && Target.CanLocalControl && Target.Queue.Count > 0 )\n\t\t\t\t\t{\n\t\t\t\t\t\t<div class=\"small-btn nav\" onclick=\"@Target.RequestClearQueue\">Clear</div>\n\t\t\t\t\t}\n\t\t\t\t</div>\n\n\t\t\t\t<div class=\"queue-list\">\n\t\t\t\t\t@if ( ShowRecent )\n\t\t\t\t\t{\n\t\t\t\t\t\t@if ( RecentLinks.All.Count == 0 )\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t<div class=\"empty\">Links you play show up here.</div>\n\t\t\t\t\t\t}\n\n\t\t\t\t\t\t@foreach ( var recent in RecentLinks.All )\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\tvar url = recent.Url;\n\t\t\t\t\t\t\t<div class=\"queue-item recent\">\n\t\t\t\t\t\t\t\t<div class=\"info\">\n\t\t\t\t\t\t\t\t\t<div class=\"name\">@recent.Title</div>\n\t\t\t\t\t\t\t\t\t<div class=\"by\">@recent.Url</div>\n\t\t\t\t\t\t\t\t</div>\n\t\t\t\t\t\t\t\t<div class=\"dur\">@(recent.IsLive ? \"Live\" : recent.Duration > 0 ? MediaSource.FormatTime( recent.Duration ) : \"\")</div>\n\t\t\t\t\t\t\t\t@if ( Target.CanLocalControl )\n\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t<div class=\"small-btn go nav\" onclick=\"@( () => Target.RequestPlay( url ) )\"><i>play_arrow</i></div>\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t@if ( Target.CanLocalQueue )\n\t\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t\t<div class=\"small-btn go nav\" onclick=\"@( () => Target.RequestEnqueue( url ) )\"><i>playlist_add</i></div>\n\t\t\t\t\t\t\t\t}\n\t\t\t\t\t\t\t\t<div class=\"small-btn nav\" onclick=\"@( () => RecentLinks.Remove( url ) )\"><i>close</i></div>\n\t\t\t\t\t\t\t</div>\n\t\t\t\t\t\t}\n\t\t\t\t\t}\n\t\t\t\t\telse if ( Target.Queue.Count == 0 )\n\t\t\t\t\t{\n\t\t\t\t\t\t<div class=\"empty\">The queue is empty.</div>\n\t\t\t\t\t}\n\n\t\t\t\t\t@for ( int i = 0; !ShowRecent && i < Target.Queue.Count; i++ )\n\t\t\t\t\t{\n\t\t\t\t\t\tvar index = i;\n\t\t\t\t\t\tvar item = Target.Queue[i];\n\t\t\t\t\t\t<div class=\"queue-item\">\n\t\t\t\t\t\t\t<div class=\"num\">@(index + 1)</div>\n\t\t\t\t\t\t\t<div class=\"info\">\n\t\t\t\t\t\t\t\t<div class=\"name\">@item.Title</div>\n\t\t\t\t\t\t\t\t<div class=\"by\">@item.RequestedBy</div>\n\t\t\t\t\t\t\t</div>\n\t\t\t\t\t\t\t<div class=\"dur\">@(item.IsLive ? \"Live\" : item.Duration > 0 ? MediaSource.FormatTime( item.Duration ) : \"\")</div>\n\t\t\t\t\t\t\t@if ( Target.CanLocalControl )\n\t\t\t\t\t\t\t{\n\t\t\t\t\t\t\t\t<div class=\"small-btn nav\" onclick=\"@( () => Target.RequestRemove( index ) )\"><i>close</i></div>\n\t\t\t\t\t\t\t}\n\t\t\t\t\t\t</div>\n\t\t\t\t\t}\n\t\t\t\t</div>\n\t\t\t</div>\n\n\t\t\t<div class=\"footer\">\n\t\t\t\t<div class=\"btn small nav\" onclick=\"@ToggleMute\"><i>@(MediaSettings.Mute ? \"volume_off\" : \"volume_up\")</i></div>\n\t\t\t\t<SliderControl class=\"volume nav\" Min=\"@(0f)\" Max=\"@(1f)\" Step=\"@(0.01f)\" ShowValueTooltip=\"@false\" Value=\"@MediaSettings.Volume\" OnValueChanged=\"@OnVolumeChanged\"></SliderControl>\n\t\t\t\t<div class=\"btn small nav audio-mode @(MediaSettings.SpatialAudio ? \"\" : \"flat\")\" onclick=\"@ToggleSpatial\" tooltip=\"@(MediaSettings.SpatialAudio ? \"3D audio: sound comes from the screen. Click for 2D\" : \"2D audio: no direction, still quieter further away. Click for 3D\")\"><i>@(MediaSettings.SpatialAudio ? \"surround_sound\" : \"headphones\")</i><span>@(MediaSettings.SpatialAudio ? \"3D\" : \"2D\")</span></div>\n\t\t\t\t@if ( PadMode )\n\t\t\t\t{\n\t\t\t\t\t<div class=\"hint\">\n\t\t\t\t\t\t<InputGlyph Analog=\"LeftStickY\" Size=\"@InputGlyphSize.Small\"></InputGlyph><span>Move</span>\n\t\t\t\t\t\t@if ( Pad.SelectAction is not null )\n\t\t\t\t\t\t{\n\t\t\t\t\t\t\t<InputGlyph Action=\"@Pad.SelectAction\" Size=\"@InputGlyphSize.Small\"></InputGlyph><span>Select</span>\n\t\t\t\t\t\t}\n\t\t\t\t\t\t<InputGlyph Action=\"@(Pad.BackAction ?? \"use\")\" Size=\"@InputGlyphSize.Small\"></InputGlyph><span>Close</span>\n\t\t\t\t\t</div>\n\t\t\t\t}\n\t\t\t\telse\n\t\t\t\t{\n\t\t\t\t\t<div class=\"hint\"><span>Your audio (all players) \u00b7</span><InputGlyph Action=\"use\" Size=\"@InputGlyphSize.Small\"></InputGlyph><span>or</span><InputGlyph Key=\"escape\" Size=\"@InputGlyphSize.Small\"></InputGlyph><span>to close</span></div>\n\t\t\t\t}\n\t\t\t</div>\n\n\t\t</div>\n\t}\n\n</root>\n\n@code\n{\n\t/// <summary>\n\t/// The player being controlled.\n\t/// </summary>\n\tpublic MediaPlayer Target { get; set; }\n\n\t/// <summary>\n\t/// Close automatically when the camera gets further than this from the player.\n\t/// </summary>\n\tpublic float MaxDistance { get; set; } = 1500.0f;\n\n\tTextEntry UrlEntry { get; set; }\n\n\t/// <summary> The list shows the links this client has played (<see cref=\"RecentLinks\"/>) instead of the queue. </summary>\n\tbool ShowRecent { get; set; }\n\n\tRealTimeSince sinceOpened = 0;\n\tfloat? pendingSeek;\n\tRealTimeSince sincePendingSeek;\n\n\tfloat SeekValue => pendingSeek ?? Target?.CurrentTime ?? 0;\n\n\tstring TimeText\n\t{\n\t\tget\n\t\t{\n\t\t\tif ( Target is null || !Target.HasMedia ) return \"\";\n\t\t\tif ( Target.IsLive ) return \"Live\";\n\t\t\tvar cur = MediaSource.FormatTime( Target.CurrentTime );\n\t\t\tvar s = Target.MediaDuration > 0 ? $\"{cur} / {MediaSource.FormatTime( Target.MediaDuration )}\" : cur;\n\t\t\treturn Target.Paused ? $\"{s} (paused)\" : s;\n\t\t}\n\t}\n\n\t/// <summary>\n\t/// Open the remote for this player, or close it if it's already open for it.\n\t/// </summary>\n\tpublic static void Toggle( MediaPlayer target )\n\t{\n\t\tif ( !target.IsValid() || Application.IsDedicatedServer ) return;\n\n\t\tvar existing = target.Scene.GetAllComponents<MediaRemote>().FirstOrDefault();\n\t\tif ( existing.IsValid() )\n\t\t{\n\t\t\tif ( existing.Target == target )\n\t\t\t{\n\t\t\t\texisting.Close();\n\t\t\t\treturn;\n\t\t\t}\n\n\t\t\texisting.Target = target;\n\t\t\texisting.sinceOpened = 0;\n\t\t\texisting.StateHasChanged();\n\t\t\treturn;\n\t\t}\n\n\t\tvar go = target.Scene.CreateObject();\n\t\tgo.Name = \"bimp remote\";\n\t\tgo.NetworkMode = NetworkMode.Never;\n\t\tgo.Flags |= GameObjectFlags.NotSaved | GameObjectFlags.Hidden;\n\n\t\tvar screen = go.Components.Create<ScreenPanel>();\n\t\tscreen.ZIndex = 200;\n\n\t\tvar remote = go.Components.Create<MediaRemote>();\n\t\tremote.Target = target;\n\t}\n\n\t/// <summary>\n\t/// Is a remote open right now? Games can use this to e.g. hide their crosshair.\n\t/// </summary>\n\tpublic static bool IsOpen( Scene scene ) => scene?.GetAllComponents<MediaRemote>().Any() ?? false;\n\n\tpublic void Close()\n\t{\n\t\tGameObject.Destroy();\n\t}\n\n\tprotected override void OnTreeFirstBuilt()\n\t{\n\t\tbase.OnTreeFirstBuilt();\n\t\tUrlEntry?.Focus();\n\t}\n\n\t//\n\t// Controller: the D-pad / left stick moves a highlight between the controls (the \"nav\" ones), A presses,\n\t// left/right adjusts sliders and choices, B closes. See PadNavigation.\n\t//\n\n\t/// <summary> Played with a controller right now (the engine switches as soon as they touch the keyboard or pad). </summary>\n\tbool PadMode => Input.UsingController;\n\n\treadonly PadNavigation Pad = new();\n\n\t/// <summary> The engine's virtual cursor (left stick moves a pointer) is off while we navigate; this was its setting. </summary>\n\tbool? virtualCursorWas;\n\n\t/// <summary>\n\t/// The very first thing in the frame, before anything (fixed update included) reads input: navigate, then suppress the\n\t/// frame's input, so the pad doesn't also move, jump or duck the player behind the remote.\n\t/// </summary>\n\tvoid Component.ISceneStage.Start()\n\t{\n\t\tif ( !PadMode || !Target.IsValid() || Panel is null )\n\t\t{\n\t\t\tRestoreVirtualCursor();\n\t\t\treturn;\n\t\t}\n\n\t\tvirtualCursorWas ??= Input.EnableVirtualCursor;\n\t\tInput.EnableVirtualCursor = false;\n\n\t\tPad.Press ??= PressControl;\n\t\tPad.Adjust ??= AdjustControl;\n\t\tPad.Update( Panel );\n\n\t\t// B closes (unless it just closed a drop down's menu), and USE again, as on keyboard\n\t\tvar close = sinceOpened > 0.2f && (Pad.BackPressed || Input.Pressed( \"use\" ));\n\t\tInput.Suppressed = true;\n\t\tif ( close ) Close();\n\t}\n\n\t/// <summary> The window's width in the stylesheet (.window), at 1080p. </summary>\n\tconst float WindowWidth = 760;\n\n\t/// <summary>\n\t/// The screen panel already scales the remote with the screen height (as if every screen were 1080 high); this sizes\n\t/// it up from there by bimp_ui_scale - at its design size it was small at every resolution - but never wider than\n\t/// 90% of the screen.\n\t/// </summary>\n\tvoid FitToScreen()\n\t{\n\t\tif ( Components.Get<ScreenPanel>() is not { } screen ) return;\n\n\t\tvar heightScale = Screen.Size.y / 1080.0f;\n\t\tvar widest = heightScale > 0 ? Screen.Size.x * 0.9f / (WindowWidth * heightScale) : 1;\n\t\tvar scale = Math.Clamp( MediaSettings.RemoteScale, 0.5f, Math.Max( 0.5f, widest ) );\n\t\tif ( MathF.Abs( screen.Scale - scale ) > 0.001f ) screen.Scale = scale;\n\t}\n\n\t/// <summary> For probes: each option of an open menu, its classes, and * on the pad's selection. </summary>\n\tIEnumerable<string> MenuOptions()\n\t{\n\t\tvar top = Panel;\n\t\twhile ( top?.Parent is not null ) top = top.Parent;\n\t\tvar menu = top?.Descendants.OfType<Popup>().FirstOrDefault();\n\t\treturn menu is null ? Enumerable.Empty<string>() : menu.Children.Select( c => $\"{(c as Button)?.Text}:{c.Classes}{(c == menu.SelectedChild ? \"*\" : \"\")}\" );\n\t}\n\n\t/// <summary> For probes: the controller state and what's highlighted. </summary>\n\tinternal string DescribePad() => $\"usingController={Input.UsingController} controllers={Input.ControllerCount} select={Pad.SelectAction ?? \"-\"} back={Pad.BackAction ?? \"-\"} virtualCursor={Input.EnableVirtualCursor} screen={Screen.Size} desktopScale={Screen.DesktopScale} uiScale={Panel?.ScaleToScreen} window={Panel?.Descendants.FirstOrDefault( p => p.HasClass( \"window\" ) )?.Box.Rect} focused=[{Pad.Focused?.Classes}] menu=[{string.Join( \" | \", MenuOptions() )}]\" + '\\n' +\n\t\tstring.Join( '\\n', Panel.Descendants.Where( p => p.HasClass( \"nav\" ) ).Select( p => $\" [{p.Classes}] visible={p.IsVisible} rect={p.Box.Rect}\" ) );\n\n\t/// <summary> For probes: move the highlight as the pad would (up, down, left, right, or \"a\" to press it), and say where it went. </summary>\n\tinternal string ProbeMove( string direction )\n\t{\n\t\tif ( direction == \"a\" )\n\t\t{\n\t\t\tif ( Pad.Focused is { } focused ) PressControl( focused );\n\t\t\treturn $\"a on [{Pad.Focused?.Classes}]\";\n\t\t}\n\n\t\tvar d = direction switch { \"up\" => new Vector2( 0, -1 ), \"down\" => new Vector2( 0, 1 ), \"left\" => new Vector2( -1, 0 ), _ => new Vector2( 1, 0 ) };\n\t\tPad.ProbeStep( Panel, d );\n\t\treturn $\"{direction} -> [{Pad.Focused?.Classes}] volume={MediaSettings.Volume:0.00}\";\n\t}\n\n\tvoid RestoreVirtualCursor()\n\t{\n\t\tif ( virtualCursorWas is not { } was ) return;\n\t\tInput.EnableVirtualCursor = was;\n\t\tvirtualCursorWas = null;\n\t}\n\n\tprotected override void OnDestroy()\n\t{\n\t\tRestoreVirtualCursor();\n\t\tbase.OnDestroy();\n\t}\n\n\t/// <summary> A on a control: a drop down opens its menu (navigated by PadNavigation), a button is clicked. Sliders only take left/right. </summary>\n\tvoid PressControl( Panel control )\n\t{\n\t\tif ( control.HasClass( \"seek\" ) || control.HasClass( \"volume\" ) ) return;\n\n\t\tif ( control is DropDown dropDown )\n\t\t{\n\t\t\tdropDown.Open();\n\t\t\treturn;\n\t\t}\n\n\t\tcontrol.CreateEvent( new MousePanelEvent( \"onclick\", control, \"mouseleft\" ) );\n\t}\n\n\t/// <summary> Left/right on a slider: seek by 10 s, volume by 5%. Anything else moves left/right. </summary>\n\tbool AdjustControl( Panel control, int step )\n\t{\n\t\tif ( control.HasClass( \"seek\" ) )\n\t\t{\n\t\t\tOnSeekChanged( Math.Clamp( SeekValue + step * 10, 0, Target.MediaDuration ) );\n\t\t\treturn true;\n\t\t}\n\n\t\tif ( control.HasClass( \"volume\" ) )\n\t\t{\n\t\t\tOnVolumeChanged( MathF.Round( (MediaSettings.Volume + step * 0.05f) * 20 ) / 20 );\n\t\t\treturn true;\n\t\t}\n\n\t\treturn false;\n\t}\n\n\tprotected override void OnUpdate()\n\t{\n\t\tif ( !Target.IsValid() )\n\t\t{\n\t\t\tClose();\n\t\t\treturn;\n\t\t}\n\n\t\tMenuStyle.Apply( Panel );\n\t\tFitToScreen();\n\n\t\tif ( Input.EscapePressed )\n\t\t{\n\t\t\tInput.EscapePressed = false;\n\t\t\tClose();\n\t\t\treturn;\n\t\t}\n\n\t\t// USE again closes it (not while typing - a focused text entry takes the keys)\n\t\tif ( sinceOpened > 0.2f && Input.Pressed( \"use\" ) && UseHandled.TryConsume() )\n\t\t{\n\t\t\tClose();\n\t\t\treturn;\n\t\t}\n\n\t\tif ( Scene.Camera is not null && Scene.Camera.WorldPosition.Distance( Target.WorldPosition ) > MaxDistance )\n\t\t{\n\t\t\tClose();\n\t\t\treturn;\n\t\t}\n\n\t\t// Seek once the slider has settled, rather than spamming the host while dragging\n\t\tif ( pendingSeek.HasValue && sincePendingSeek > 0.3f )\n\t\t{\n\t\t\tTarget.RequestSeek( pendingSeek.Value );\n\t\t\tpendingSeek = null;\n\t\t}\n\t}\n\n\tvoid OnSeekChanged( float value )\n\t{\n\t\tif ( Target is null || !Target.CanLocalControl ) return;\n\t\tif ( MathF.Abs( value - Target.CurrentTime ) < 1.5f && !pendingSeek.HasValue ) return; // our own refresh\n\n\t\tpendingSeek = value;\n\t\tsincePendingSeek = 0;\n\t}\n\n\t//\n\t// Resolution / audio track. These are this player's own preferences (saved convars), applied to every\n\t// media player - picking one reloads the local stream at the same synced time.\n\t//\n\n\tstring QualityValue => MediaSettings.Quality <= 0 ? \"0\" : Target.SelectedQuality.ToString();\n\n\tList<Option> BuildQualityOptions()\n\t{\n\t\tvar auto = $\"Auto (up to {MediaSettings.MaxVideoHeight}p)\";\n\t\tvar list = new List<Option> { new Option( auto, \"0\" ) };\n\t\tforeach ( var h in Target.AvailableQualities )\n\t\t\tlist.Add( new Option( MediaStreamOptions.QualityLabel( h ), h.ToString() ) );\n\t\treturn list;\n\t}\n\n\tvoid OnQualityChanged( string value )\n\t{\n\t\tif ( !int.TryParse( value, out var h ) ) return;\n\t\tif ( h == (MediaSettings.Quality <= 0 ? 0 : Target.SelectedQuality) ) return;\n\t\tMediaSettings.Quality = h;\n\t}\n\n\tstring AudioTrackValue\n\t{\n\t\tget\n\t\t{\n\t\t\tvar selected = Target.SelectedAudioTrack ?? Target.AvailableAudioTracks.FirstOrDefault( t => t.IsDefault );\n\t\t\treturn selected?.Id ?? \"\";\n\t\t}\n\t}\n\n\tList<Option> BuildAudioTrackOptions()\n\t{\n\t\treturn Target.AvailableAudioTracks\n\t\t\t.Select( t => new Option( t.IsDefault && !t.Name.Contains( \"original\", StringComparison.OrdinalIgnoreCase ) ? $\"{t.Name} (original)\" : t.Name, t.Id ) )\n\t\t\t.ToList();\n\t}\n\n\tvoid OnAudioTrackChanged( string id )\n\t{\n\t\tif ( id == AudioTrackValue ) return;\n\t\tvar track = Target.AvailableAudioTracks.FirstOrDefault( t => t.Id == id );\n\t\t// picking the original clears the preference, so dubbed videos default to original again\n\t\tMediaSettings.AudioLanguage = track is null || track.IsDefault ? \"\" : track.Id;\n\t}\n\n\t//\n\t// Captions: off, or a language. The choice is this player's own (saved convars) and applies to every media player.\n\t// Auto-generated tracks are only offered for languages without a written one - that's what picking a language gets.\n\t//\n\n\tIEnumerable<MediaCaptionTrack> ListedCaptionTracks\n\t\t=> Target.AvailableCaptionTracks.Where( t => !t.IsAuto || !Target.AvailableCaptionTracks.Any( w => !w.IsAuto && w.Lang == t.Lang ) );\n\n\tstring CaptionValue => Target.SelectedCaptionTrack?.Id ?? \"\";\n\n\tList<Option> BuildCaptionOptions()\n\t{\n\t\tvar list = new List<Option> { new Option( \"Captions off\", \"\" ) };\n\t\tforeach ( var t in ListedCaptionTracks )\n\t\t\tlist.Add( new Option( t.IsAuto && !t.Name.Contains( \"auto\", StringComparison.OrdinalIgnoreCase ) ? $\"{t.Name} (auto-generated)\" : t.Name, t.Id ) );\n\t\treturn list;\n\t}\n\n\tvoid OnCaptionChanged( string id )\n\t{\n\t\tif ( id == CaptionValue ) return;\n\n\t\tvar track = Target.AvailableCaptionTracks.FirstOrDefault( t => t.Id == id );\n\t\tMediaSettings.Subtitles = track is not null;\n\t\tif ( track is not null ) MediaSettings.SubtitleLanguage = track.Lang;\n\t}\n\n\tstring PlayingAsText\n\t{\n\t\tget\n\t\t{\n\t\t\tvar b = Target.Backend;\n\t\t\tif ( b is null || !b.Loaded ) return \"Loading...\";\n\t\t\treturn b.HasVideo ? $\"Playing {b.Height}p\" : \"\";\n\t\t}\n\t}\n\n\tvoid OnVolumeChanged( float value )\n\t{\n\t\tMediaSettings.Volume = value.Clamp( 0, 1 );\n\t\tif ( value > 0 ) MediaSettings.Mute = false;\n\t}\n\n\tvoid ToggleMute()\n\t{\n\t\tMediaSettings.Mute = !MediaSettings.Mute;\n\t}\n\n\t/// <summary> 3D audio from every media player, or flat 2D for this player (bimp_spatial). </summary>\n\tvoid ToggleSpatial()\n\t{\n\t\tMediaSettings.SpatialAudio = !MediaSettings.SpatialAudio;\n\t}\n\n\tstring TakeUrl()\n\t{\n\t\tvar text = UrlEntry?.Text?.Trim();\n\t\tif ( UrlEntry is not null ) UrlEntry.Text = \"\";\n\t\treturn string.IsNullOrWhiteSpace( text ) ? null : text;\n\t}\n\n\tvoid OnSubmit()\n\t{\n\t\t// Enter plays right away if we can, otherwise queues\n\t\tif ( Target.CanLocalControl && !Target.HasMedia ) PlayNow();\n\t\telse if ( Target.CanLocalQueue ) AddToQueue();\n\t\telse PlayNow();\n\t}\n\n\tvoid PlayNow()\n\t{\n\t\tvar url = TakeUrl();\n\t\tif ( url is null ) return;\n\t\tTarget.RequestPlay( url );\n\t}\n\n\tvoid AddToQueue()\n\t{\n\t\tvar url = TakeUrl();\n\t\tif ( url is null ) return;\n\t\tTarget.RequestEnqueue( url );\n\t}\n\n\tprotected override int BuildHash()\n\t{\n\t\tif ( Target is null ) return 0;\n\n\t\treturn HashCode.Combine(\n\t\t\tHashCode.Combine( Target.PlayId, Target.Title, Target.Paused, Target.Status, Target.LocalNotice, (int)Target.CurrentTime, Target.MediaDuration ),\n\t\t\tHashCode.Combine( QueueHash(), MediaSettings.Mute, MediaSettings.Volume, MediaSettings.SpatialAudio, Target.CanLocalControl, pendingSeek, PadMode ),\n\t\t\tHashCode.Combine( Target.Qualities, Target.AudioTracks, MediaSettings.Quality, MediaSettings.AudioLanguage, PlayingAsText, ShowRecent, RecentLinks.Version ),\n\t\t\tHashCode.Combine( Target.CaptionTracks, MediaSettings.Subtitles, MediaSettings.SubtitleLanguage )\n\t\t);\n\t}\n\n\tint QueueHash()\n\t{\n\t\tvar h = new HashCode();\n\t\th.Add( Target.Queue.Count );\n\t\tforeach ( var item in Target.Queue ) h.Add( item?.Url );\n\t\treturn h.ToHashCode();\n\t}\n}\n"
},
{
"Ident": "rexi.bimp",
"Path": "MediaSpeaker.razor",
"FileName": "MediaSpeaker.razor",
"PackageType": "library",
"CodeKind": "Game",
"AssetVersionId": 399292,
"IsPrivate": false,
"Code": "@using System\n@using Sandbox\n@using Sandbox.UI\n@namespace Bimp\n@inherits PanelComponent\n@implements Component.ExecuteInEditor\n\n@*\n\tA small display for audio-only players (speakers, radios, jukeboxes).\n\tPut this on a GameObject with a WorldPanel (PanelSize ~800x400).\n*@\n\n<root class=\"mediaspeaker @(Player?.HasMedia == true ? \"playing\" : \"idle\")\">\n\n\t<div class=\"header\">\n\t\t<i>@(Player?.HasMedia == true ? \"graphic_eq\" : \"speaker\")</i>\n\t\t@if ( ShowUseHint )\n\t\t{\n\t\t\t<div class=\"title hint\"><span>Press</span><InputGlyph Action=\"use\"></InputGlyph><span>to play something</span></div>\n\t\t}\n\t\telse\n\t\t{\n\t\t\t<ScrollingText class=\"title\" Text=\"@TitleText\"></ScrollingText>\n\t\t}\n\t</div>\n\n\t<SpectrumBars Player=\"@Player\"></SpectrumBars>\n\n\t<div class=\"footer\">\n\t\t<div class=\"time\">@TimeText</div>\n\t\t@if ( Player is not null && Player.Queue.Count > 0 )\n\t\t{\n\t\t\t<div class=\"queue\"><i>queue_music</i> @Player.Queue.Count</div>\n\t\t}\n\t</div>\n\n\t@if ( !string.IsNullOrWhiteSpace( Player?.LocalNotice ) )\n\t{\n\t\t<div class=\"notice\">@Player.LocalNotice</div>\n\t}\n\n</root>\n\n@code\n{\n\t/// <summary>\n\t/// The player this display shows. Found automatically in this object or its parents if unset.\n\t/// </summary>\n\t[Property] public MediaPlayer Player { get; set; }\n\n\t/// <summary>\n\t/// Scale the UI to fit the WorldPanel's PanelSize instead of getting cut off. Sets the WorldPanel's RenderScale.\n\t/// </summary>\n\t[Property, Group( \"Layout\" )] public bool FitToPanel { get; set; } = true;\n\n\t/// <summary>\n\t/// The panel size the UI is designed for.\n\t/// </summary>\n\t[Property, Group( \"Layout\" ), ShowIf( nameof( FitToPanel ), true )] public Vector2 DesignSize { get; set; } = new Vector2( 560, 280 );\n\n\tstring TitleText\n\t{\n\t\tget\n\t\t{\n\t\t\tif ( Player is null ) return \"\";\n\t\t\tif ( Player.HasMedia ) return Player.Title;\n\t\t\tif ( !string.IsNullOrWhiteSpace( Player.Status ) ) return Player.Status;\n\t\t\treturn \"\";\n\t\t}\n\t}\n\n\t/// <summary> Nothing playing or happening: show how to start something, with the USE button's glyph. </summary>\n\tbool ShowUseHint => Player is not null && !Player.HasMedia && string.IsNullOrWhiteSpace( Player.Status );\n\n\tstring TimeText\n\t{\n\t\tget\n\t\t{\n\t\t\tif ( Player is null || !Player.HasMedia ) return \"\";\n\t\t\tif ( Player.IsLive ) return \"Live\";\n\t\t\tvar cur = MediaSource.FormatTime( Player.CurrentTime );\n\t\t\tvar s = Player.MediaDuration > 0 ? $\"{cur} / {MediaSource.FormatTime( Player.MediaDuration )}\" : cur;\n\t\t\treturn Player.Paused ? $\"{s} (paused)\" : s;\n\t\t}\n\t}\n\n\tprotected override void OnUpdate()\n\t{\n\t\tif ( FitToPanel ) PanelFit.Apply( this, DesignSize );\n\t\tif ( Scene.IsEditor ) return;\n\n\t\tPlayer ??= GetComponentInParent<MediaPlayer>();\n\t}\n\n\tprotected override int BuildHash()\n\t{\n\t\tif ( Player is null ) return 0;\n\t\treturn HashCode.Combine( Player.PlayId, Player.Title, Player.Status, Player.Paused, (int)Player.CurrentTime, Player.Queue.Count, Player.LocalNotice );\n\t}\n}\n"
},
{
"Ident": "rexi.bimp",
"Path": "Code/MediaInteract.cs",
"FileName": "MediaInteract.cs",
"PackageType": "library",
"CodeKind": "Game",
"AssetVersionId": 399292,
"IsPrivate": false,
"Code": "namespace Bimp;\n\n/// <summary>\n/// Lets players use the media player with the USE key (works with the engine's <see cref=\"PlayerController\"/>,\n/// or anything else that calls <see cref=\"Component.IPressable\"/>). Needs a collider on this GameObject or a child.\n/// <para>\n/// Aiming at one of the screen's on-screen buttons while the controls are showing presses that button.\n/// Anywhere else (or while the controls are hidden) opens the full remote.\n/// </para>\n/// </summary>\n[Title( \"Media Player Interact\" ), Category( \"Media\" ), Icon( \"touch_app\" )]\npublic sealed class MediaInteract : Component, Component.IPressable\n{\n\t/// <summary>\n\t/// The player to control. Found automatically in this object or its parents if unset.\n\t/// </summary>\n\t[Property] public MediaPlayer Player { get; set; }\n\n\t/// <summary>\n\t/// The screen whose on-screen buttons USE can press. Found automatically in this object or its children if unset.\n\t/// </summary>\n\t[Property] public MediaScreen Screen { get; set; }\n\n\tMediaPlayer Target => Player.IsValid() ? Player : (Player = GetComponentInParent<MediaPlayer>());\n\tMediaScreen TargetScreen => Screen.IsValid() ? Screen : (Screen = GetComponentInChildren<MediaScreen>());\n\n\tpublic bool CanPress( IPressable.Event e ) => Target.IsValid();\n\n\tpublic bool Press( IPressable.Event e )\n\t{\n\t\tif ( !Target.IsValid() ) return false;\n\n\t\t// The screen may already have handled this USE press itself this frame\n\t\tif ( !UseHandled.TryConsume() ) return true;\n\n\t\t// Press runs on the client that pressed, so this only affects them (button presses go through the\n\t\t// player's permission-checked host requests)\n\t\tvar screen = TargetScreen;\n\t\tif ( screen.IsValid() && screen.TryPressAimedControl() )\n\t\t\treturn true;\n\n\t\tMediaRemote.Toggle( Target );\n\t\treturn true;\n\t}\n\n\tpublic IPressable.Tooltip? GetTooltip( IPressable.Event e )\n\t{\n\t\tif ( !Target.IsValid() ) return null;\n\n\t\tvar title = Target.HasMedia ? Target.Title : \"Media Player\";\n\t\tvar icon = Target.AudioOnlyPlayer ? \"speaker\" : \"smart_display\";\n\n\t\tvar action = TargetScreen.IsValid() ? TargetScreen.AimedActionName : null;\n\t\tif ( action is not null )\n\t\t\treturn new IPressable.Tooltip( action, \"touch_app\", title );\n\n\t\treturn new IPressable.Tooltip( title, icon, \"Open controls\" );\n\t}\n}\n"
},
{
"Ident": "rexi.bimp",
"Path": "Code/MediaSpeaker.razor",
"FileName": "MediaSpeaker.razor",
"PackageType": "library",
"CodeKind": "Game",
"AssetVersionId": 399292,
"IsPrivate": false,
"Code": "@using System\n@using Sandbox\n@using Sandbox.UI\n@namespace Bimp\n@inherits PanelComponent\n@implements Component.ExecuteInEditor\n\n@*\n\tA small display for audio-only players (speakers, radios, jukeboxes).\n\tPut this on a GameObject with a WorldPanel (PanelSize ~800x400).\n*@\n\n<root class=\"mediaspeaker @(Player?.HasMedia == true ? \"playing\" : \"idle\")\">\n\n\t<div class=\"header\">\n\t\t<i>@(Player?.HasMedia == true ? \"graphic_eq\" : \"speaker\")</i>\n\t\t@if ( ShowUseHint )\n\t\t{\n\t\t\t<div class=\"title hint\"><span>Press</span><InputGlyph Action=\"use\"></InputGlyph><span>to play something</span></div>\n\t\t}\n\t\telse\n\t\t{\n\t\t\t<ScrollingText class=\"title\" Text=\"@TitleText\"></ScrollingText>\n\t\t}\n\t</div>\n\n\t<SpectrumBars Player=\"@Player\"></SpectrumBars>\n\n\t<div class=\"footer\">\n\t\t<div class=\"time\">@TimeText</div>\n\t\t@if ( Player is not null && Player.Queue.Count > 0 )\n\t\t{\n\t\t\t<div class=\"queue\"><i>queue_music</i> @Player.Queue.Count</div>\n\t\t}\n\t</div>\n\n\t@if ( !string.IsNullOrWhiteSpace( Player?.LocalNotice ) )\n\t{\n\t\t<div class=\"notice\">@Player.LocalNotice</div>\n\t}\n\n</root>\n\n@code\n{\n\t/// <summary>\n\t/// The player this display shows. Found automatically in this object or its parents if unset.\n\t/// </summary>\n\t[Property] public MediaPlayer Player { get; set; }\n\n\t/// <summary>\n\t/// Scale the UI to fit the WorldPanel's PanelSize instead of getting cut off. Sets the WorldPanel's RenderScale.\n\t/// </summary>\n\t[Property, Group( \"Layout\" )] public bool FitToPanel { get; set; } = true;\n\n\t/// <summary>\n\t/// The panel size the UI is designed for.\n\t/// </summary>\n\t[Property, Group( \"Layout\" ), ShowIf( nameof( FitToPanel ), true )] public Vector2 DesignSize { get; set; } = new Vector2( 560, 280 );\n\n\tstring TitleText\n\t{\n\t\tget\n\t\t{\n\t\t\tif ( Player is null ) return \"\";\n\t\t\tif ( Player.HasMedia ) return Player.Title;\n\t\t\tif ( !string.IsNullOrWhiteSpace( Player.Status ) ) return Player.Status;\n\t\t\treturn \"\";\n\t\t}\n\t}\n\n\t/// <summary> Nothing playing or happening: show how to start something, with the USE button's glyph. </summary>\n\tbool ShowUseHint => Player is not null && !Player.HasMedia && string.IsNullOrWhiteSpace( Player.Status );\n\n\tstring TimeText\n\t{\n\t\tget\n\t\t{\n\t\t\tif ( Player is null || !Player.HasMedia ) return \"\";\n\t\t\tif ( Player.IsLive ) return \"Live\";\n\t\t\tvar cur = MediaSource.FormatTime( Player.CurrentTime );\n\t\t\tvar s = Player.MediaDuration > 0 ? $\"{cur} / {MediaSource.FormatTime( Player.MediaDuration )}\" : cur;\n\t\t\treturn Player.Paused ? $\"{s} (paused)\" : s;\n\t\t}\n\t}\n\n\tprotected override void OnUpdate()\n\t{\n\t\tif ( FitToPanel ) PanelFit.Apply( this, DesignSize );\n\t\tif ( Scene.IsEditor ) return;\n\n\t\tPlayer ??= GetComponentInParent<MediaPlayer>();\n\t}\n\n\tprotected override int BuildHash()\n\t{\n\t\tif ( Player is null ) return 0;\n\t\treturn HashCode.Combine( Player.PlayId, Player.Title, Player.Status, Player.Paused, (int)Player.CurrentTime, Player.Queue.Count, Player.LocalNotice );\n\t}\n}\n"
},
{
"Ident": "rexi.bimp",
"Path": "Code/MediaSpeaker.razor.scss",
"FileName": "MediaSpeaker.razor.scss",
"PackageType": "library",
"CodeKind": "Game",
"AssetVersionId": 399292,
"IsPrivate": false,
"Code": ".mediaspeaker\n{\n\twidth: 100%;\n\theight: 100%;\n\tflex-direction: column;\n\tpadding: 28px 36px;\n\tborder-radius: 24px;\n\tbackground: linear-gradient( to bottom, #151a24, #07090d );\n\tfont-family: Poppins;\n\tcolor: white;\n\tposition: relative;\n\n\ti\n\t{\n\t\tfont-family: \"Material Icons\";\n\t}\n\n\t.header\n\t{\n\t\tflex-direction: row;\n\t\talign-items: center;\n\t\tgap: 16px;\n\n\t\ti\n\t\t{\n\t\t\tfont-size: 56px;\n\t\t\tcolor: #3fa9ff;\n\t\t}\n\n\t\t.title\n\t\t{\n\t\t\tflex-grow: 1;\n\t\t\tflex-basis: 0;\n\t\t\tfont-size: 40px;\n\t\t\tfont-weight: 700;\n\t\t}\n\n\t\t.hint\n\t\t{\n\t\t\tflex-direction: row;\n\t\t\talign-items: center;\n\t\t\tgap: 10px;\n\t\t\tfont-size: 28px;\n\t\t\tfont-weight: 600;\n\t\t\topacity: 0.8;\n\t\t\twhite-space: nowrap;\n\n\t\t\tspan\n\t\t\t{\n\t\t\t\tflex-shrink: 0;\n\t\t\t}\n\t\t}\n\t}\n\n\t.spectrumbars\n\t{\n\t\tflex-grow: 1;\n\t\tmargin: 20px 0;\n\t}\n\n\t.footer\n\t{\n\t\tflex-direction: row;\n\t\tfont-size: 28px;\n\t\topacity: 0.75;\n\n\t\t.time\n\t\t{\n\t\t\tflex-grow: 1;\n\t\t\t// every digit the same width, so it doesn't jiggle as it counts (Poppins has no tabular digits, Inter does)\n\t\t\tfont-family: Inter;\n\t\t\tfont-variant-numeric: tabular-nums;\n\t\t}\n\n\t\t.queue\n\t\t{\n\t\t\talign-items: center;\n\t\t\tgap: 6px;\n\t\t}\n\t}\n\n\t.notice\n\t{\n\t\tposition: absolute;\n\t\tleft: 20px;\n\t\tright: 20px;\n\t\tbottom: 20px;\n\t\tpadding: 12px 20px;\n\t\tborder-radius: 12px;\n\t\tbackground-color: rgba( 180, 40, 40, 0.92 );\n\t\tfont-size: 26px;\n\t}\n\n\t&.idle .spectrumbars\n\t{\n\t\topacity: 0.3;\n\t}\n}\n\n.spectrumbars\n{\n\tflex-direction: row;\n\talign-items: flex-end;\n\tgap: 6px;\n\n\t.bar\n\t{\n\t\tflex-grow: 1;\n\t\theight: 3%;\n\t\tborder-radius: 4px 4px 0 0;\n\t\tbackground: linear-gradient( to top, #3fa9ff, #a36bff );\n\t}\n}\n"
},
{
"Ident": "rexi.bimp",
"Path": "Code/Resolver/Live/AacDecoder.cs",
"FileName": "AacDecoder.cs",
"PackageType": "library",
"CodeKind": "Game",
"AssetVersionId": 399292,
"IsPrivate": false,
"Code": "using System;\r\n\r\nnamespace Bimp.Resolver.Live;\r\n\r\n/// <summary>\r\n/// AAC-LC decoder (ISO/IEC 14496-3 GA, 1024-sample frames): one raw access unit in, interleaved 16-bit PCM out.\r\n/// The engine only decodes AAC inside finished MP4 files, which ties live audio to the segment latency - this\r\n/// lets low-latency live streams play their AAC beside the video, like G.711 (see <see cref=\"LivePcmAudio\"/>).\r\n/// <para>\r\n/// Supports SCE/CPE/LFE, section and scalefactor data, pulse data, M/S and intensity stereo, PNS, TNS, and the\r\n/// long / start / eight-short / stop windows with sine and KBD shapes. HE-AAC decodes its AAC-LC core (half\r\n/// band, SBR ignored). Main/LTP prediction and coupling channels aren't supported. Output is mono or stereo\r\n/// (anything with more channels is folded down to its front three).\r\n/// </para>\r\n/// </summary>\r\npublic sealed class AacDecoder\r\n{\r\n\tconst int ZeroHcb = 0, EscHcb = 11, NoiseHcb = 13, IntensityHcb2 = 14, IntensityHcb = 15;\r\n\tconst int OnlyLong = 0, LongStart = 1, EightShort = 2, LongStop = 3;\r\n\r\n\tpublic int SampleRate { get; }\r\n\r\n\t/// <summary> Output channels: 1 or 2. </summary>\r\n\tpublic int Channels { get; }\r\n\r\n\treadonly int rateIndex;\r\n\treadonly short[] swbLong, swbShort;\r\n\treadonly int numSwbLong, numSwbShort;\r\n\r\n\t// per channel state: the second half of the last frame, and its window shape\r\n\treadonly float[][] overlap = { new float[1024], new float[1024], new float[1024], new float[1024] };\r\n\treadonly int[] previousShape = new int[4];\r\n\r\n\t// scratch\r\n\treadonly float[][] spectrum = { new float[1024], new float[1024] };\r\n\treadonly float[] timeBuffer = new float[2048];\r\n\treadonly float[][] pcm = { new float[1024], new float[1024], new float[1024], new float[1024] };\r\n\tuint noiseState = 0x1f2e3d4c;\r\n\treadonly Ics icsA = new(), icsB = new();\r\n\r\n\tpublic AacDecoder( AacConfig config )\r\n\t{\r\n\t\tvar asc = config.Asc;\r\n\t\tvar r = new BitReader( asc );\r\n\t\tvar objectType = ReadObjectType( r );\r\n\t\tvar index = r.Read( 4 );\r\n\t\tvar rate = index == 15 ? r.Read( 24 ) : AacConfig.RateOf( index );\r\n\t\tvar channelConfig = r.Read( 4 );\r\n\r\n\t\t// HE-AAC signalled explicitly: the core is described after the SBR rate\r\n\t\tif ( objectType is 5 or 29 )\r\n\t\t{\r\n\t\t\tvar extIndex = r.Read( 4 );\r\n\t\t\tif ( extIndex == 15 ) r.Read( 24 );\r\n\t\t\tobjectType = ReadObjectType( r );\r\n\t\t}\r\n\t\tif ( objectType != 2 ) throw new NotSupportedException( $\"AAC object type {objectType} (only AAC-LC)\" );\r\n\t\tif ( r.Read( 1 ) == 1 ) throw new NotSupportedException( \"960-sample AAC frames\" );\r\n\r\n\t\trateIndex = index == 15 ? NearestRateIndex( rate ) : index;\r\n\t\tSampleRate = rate > 0 ? rate : AacConfig.RateOf( rateIndex );\r\n\t\tChannels = channelConfig == 1 ? 1 : 2;\r\n\r\n\t\t(swbLong, swbShort) = rateIndex switch\r\n\t\t{\r\n\t\t\t0 or 1 => (AacTables.Sfb96_1024, AacTables.Sfb96_128),\r\n\t\t\t2 => (AacTables.Sfb64_1024, AacTables.Sfb64_128),\r\n\t\t\t3 or 4 => (AacTables.Sfb48_1024, AacTables.Sfb48_128),\r\n\t\t\t5 => (AacTables.Sfb32_1024, AacTables.Sfb48_128),\r\n\t\t\t6 or 7 => (AacTables.Sfb24_1024, AacTables.Sfb24_128),\r\n\t\t\t8 or 9 or 10 => (AacTables.Sfb16_1024, AacTables.Sfb16_128),\r\n\t\t\t_ => (AacTables.Sfb8_1024, AacTables.Sfb8_128),\r\n\t\t};\r\n\t\tnumSwbLong = swbLong.Length - 1;\r\n\t\tnumSwbShort = swbShort.Length - 1;\r\n\t}\r\n\r\n\tstatic int ReadObjectType( BitReader r )\r\n\t{\r\n\t\tvar t = r.Read( 5 );\r\n\t\treturn t == 31 ? 32 + r.Read( 6 ) : t;\r\n\t}\r\n\r\n\tstatic int NearestRateIndex( int rate )\r\n\t{\r\n\t\tint best = 3, bestDiff = int.MaxValue;\r\n\t\tfor ( int i = 0; i < 12; i++ )\r\n\t\t{\r\n\t\t\tvar d = Math.Abs( AacConfig.RateOf( i ) - rate );\r\n\t\t\tif ( d < bestDiff ) { best = i; bestDiff = d; }\r\n\t\t}\r\n\t\treturn best;\r\n\t}\r\n\r\n\t/// <summary>\r\n\t/// Decode one access unit (a raw_data_block) to 1024 interleaved samples per channel. Null if the frame can't be\r\n\t/// decoded (it's skipped, the next one starts clean enough).\r\n\t/// </summary>\r\n\tpublic short[] Decode( byte[] frame )\r\n\t{\r\n\t\tvar r = new BitReader( frame );\r\n\t\tvar outputs = 0; // decoded channels, in pcm[]\r\n\t\tint centre = -1, left = -1, right = -1;\r\n\r\n\t\ttry\r\n\t\t{\r\n\t\t\twhile ( r.BitsLeft >= 3 )\r\n\t\t\t{\r\n\t\t\t\tvar id = r.Read( 3 );\r\n\t\t\t\tif ( id == 7 ) break; // END\r\n\r\n\t\t\t\tswitch ( id )\r\n\t\t\t\t{\r\n\t\t\t\t\tcase 0: // SCE\r\n\t\t\t\t\tcase 3: // LFE\r\n\t\t\t\t\t\tr.Read( 4 );\r\n\t\t\t\t\t\tif ( outputs < 4 )\r\n\t\t\t\t\t\t{\r\n\t\t\t\t\t\t\tvar ics = icsA;\r\n\t\t\t\t\t\t\tDecodeIcs( r, ics, false );\r\n\t\t\t\t\t\t\tReconstruct( ics, spectrum[0] );\r\n\t\t\t\t\t\t\tif ( id == 0 && centre < 0 && left < 0 ) centre = outputs;\r\n\t\t\t\t\t\t\tSynthesize( ics, spectrum[0], outputs++ );\r\n\t\t\t\t\t\t}\r\n\t\t\t\t\t\telse SkipIcs( r );\r\n\t\t\t\t\t\tbreak;\r\n\r\n\t\t\t\t\tcase 1: // CPE\r\n\t\t\t\t\t\tr.Read( 4 );\r\n\t\t\t\t\t\tDecodeCpe( r, outputs );\r\n\t\t\t\t\t\tif ( left < 0 ) { left = outputs; right = outputs + 1; }\r\n\t\t\t\t\t\toutputs += 2;\r\n\t\t\t\t\t\tbreak;\r\n\r\n\t\t\t\t\tcase 4: // DSE\r\n\t\t\t\t\t\t{\r\n\t\t\t\t\t\t\tr.Read( 4 );\r\n\t\t\t\t\t\t\tvar align = r.Read( 1 );\r\n\t\t\t\t\t\t\tvar count = r.Read( 8 );\r\n\t\t\t\t\t\t\tif ( count == 255 ) count += r.Read( 8 );\r\n\t\t\t\t\t\t\tif ( align == 1 ) r.ByteAlign();\r\n\t\t\t\t\t\t\tr.Skip( count * 8 );\r\n\t\t\t\t\t\t\tbreak;\r\n\t\t\t\t\t\t}\r\n\r\n\t\t\t\t\tcase 5: // PCE\r\n\t\t\t\t\t\tSkipPce( r );\r\n\t\t\t\t\t\tbreak;\r\n\r\n\t\t\t\t\tcase 6: // FIL (SBR and other extensions: ignored)\r\n\t\t\t\t\t\t{\r\n\t\t\t\t\t\t\tvar count = r.Read( 4 );\r\n\t\t\t\t\t\t\tif ( count == 15 ) count += r.Read( 8 ) - 1;\r\n\t\t\t\t\t\t\tr.Skip( count * 8 );\r\n\t\t\t\t\t\t\tbreak;\r\n\t\t\t\t\t\t}\r\n\r\n\t\t\t\t\tdefault: // CCE: coupling isn't supported\r\n\t\t\t\t\t\treturn null;\r\n\t\t\t\t}\r\n\t\t\t}\r\n\t\t}\r\n\t\tcatch ( IndexOutOfRangeException )\r\n\t\t{\r\n\t\t\treturn null; // truncated or corrupt frame\r\n\t\t}\r\n\r\n\t\tif ( outputs == 0 ) return null;\r\n\r\n\t\tvar result = new short[1024 * Channels];\r\n\t\tif ( Channels == 1 )\r\n\t\t{\r\n\t\t\tvar src = pcm[centre >= 0 ? centre : 0];\r\n\t\t\tfor ( int i = 0; i < 1024; i++ ) result[i] = Clip( src[i] );\r\n\t\t\treturn result;\r\n\t\t}\r\n\r\n\t\tif ( left < 0 )\r\n\t\t{\r\n\t\t\t// a mono stream without channel config 1: both sides the same\r\n\t\t\tvar src = pcm[centre >= 0 ? centre : 0];\r\n\t\t\tfor ( int i = 0; i < 1024; i++ ) result[2 * i] = result[2 * i + 1] = Clip( src[i] );\r\n\t\t\treturn result;\r\n\t\t}\r\n\r\n\t\tvar l = pcm[left];\r\n\t\tvar rr = pcm[right];\r\n\t\tvar c = centre >= 0 ? pcm[centre] : null;\r\n\t\tfor ( int i = 0; i < 1024; i++ )\r\n\t\t{\r\n\t\t\tvar cc = c is null ? 0 : c[i] * 0.7071f;\r\n\t\t\tresult[2 * i] = Clip( l[i] + cc );\r\n\t\t\tresult[2 * i + 1] = Clip( rr[i] + cc );\r\n\t\t}\r\n\t\treturn result;\r\n\t}\r\n\r\n\tstatic short Clip( float v ) => (short)Math.Clamp( (int)MathF.Round( v ), short.MinValue, short.MaxValue );\r\n\r\n\t//\r\n\t// bitstream\r\n\t//\r\n\r\n\tsealed class Ics\r\n\t{\r\n\t\tpublic int GlobalGain;\r\n\t\tpublic int WindowSequence, WindowShape, MaxSfb;\r\n\t\tpublic int NumWindows, NumWindowGroups;\r\n\t\tpublic readonly int[] WindowGroupLength = new int[8];\r\n\t\tpublic readonly int[] SfbCb = new int[8 * 64];\r\n\t\tpublic readonly int[] ScaleFactors = new int[8 * 64];\r\n\t\tpublic bool PulsePresent;\r\n\t\tpublic int PulseCount, PulseStartSfb;\r\n\t\tpublic readonly int[] PulseOffset = new int[4], PulseAmp = new int[4];\r\n\t\tpublic bool TnsPresent;\r\n\t\tpublic readonly int[] TnsFilters = new int[8];\r\n\t\tpublic readonly int[] TnsCoefRes = new int[8];\r\n\t\tpublic readonly int[,] TnsLength = new int[8, 4], TnsOrder = new int[8, 4], TnsDirection = new int[8, 4];\r\n\t\tpublic readonly float[,,] TnsLpc = new float[8, 4, 32];\r\n\t\tpublic readonly int[] Quant = new int[1024];\r\n\t\tpublic bool IsLong => WindowSequence != EightShort;\r\n\t}\r\n\r\n\tvoid ReadIcsInfo( BitReader r, Ics ics )\r\n\t{\r\n\t\tr.Read( 1 ); // ics_reserved_bit\r\n\t\tics.WindowSequence = r.Read( 2 );\r\n\t\tics.WindowShape = r.Read( 1 );\r\n\t\tif ( ics.WindowSequence == EightShort )\r\n\t\t{\r\n\t\t\tics.MaxSfb = r.Read( 4 );\r\n\t\t\tvar grouping = r.Read( 7 );\r\n\t\t\tics.NumWindows = 8;\r\n\t\t\tics.NumWindowGroups = 1;\r\n\t\t\tics.WindowGroupLength[0] = 1;\r\n\t\t\tfor ( int i = 0; i < 7; i++ )\r\n\t\t\t{\r\n\t\t\t\tif ( (grouping & (1 << (6 - i))) != 0 ) ics.WindowGroupLength[ics.NumWindowGroups - 1]++;\r\n\t\t\t\telse ics.WindowGroupLength[ics.NumWindowGroups++] = 1;\r\n\t\t\t}\r\n\t\t}\r\n\t\telse\r\n\t\t{\r\n\t\t\tics.MaxSfb = r.Read( 6 );\r\n\t\t\tics.NumWindows = 1;\r\n\t\t\tics.NumWindowGroups = 1;\r\n\t\t\tics.WindowGroupLength[0] = 1;\r\n\t\t\tif ( r.Read( 1 ) == 1 ) throw new IndexOutOfRangeException( \"prediction\" ); // AAC Main only\r\n\t\t}\r\n\t\tvar numSwb = ics.IsLong ? numSwbLong : numSwbShort;\r\n\t\tif ( ics.MaxSfb > numSwb ) throw new IndexOutOfRangeException( \"max_sfb\" );\r\n\t}\r\n\r\n\tvoid DecodeIcs( BitReader r, Ics ics, bool commonWindow )\r\n\t{\r\n\t\tics.GlobalGain = r.Read( 8 );\r\n\t\tif ( !commonWindow ) ReadIcsInfo( r, ics );\r\n\t\tReadSections( r, ics );\r\n\t\tReadScaleFactors( r, ics );\r\n\r\n\t\tics.PulsePresent = r.Read( 1 ) == 1;\r\n\t\tif ( ics.PulsePresent )\r\n\t\t{\r\n\t\t\tif ( !ics.IsLong ) throw new IndexOutOfRangeException( \"pulse in short window\" );\r\n\t\t\tics.PulseCount = r.Read( 2 ) + 1;\r\n\t\t\tics.PulseStartSfb = r.Read( 6 );\r\n\t\t\tfor ( int i = 0; i < ics.PulseCount; i++ )\r\n\t\t\t{\r\n\t\t\t\tics.PulseOffset[i] = r.Read( 5 );\r\n\t\t\t\tics.PulseAmp[i] = r.Read( 4 );\r\n\t\t\t}\r\n\t\t}\r\n\r\n\t\tics.TnsPresent = r.Read( 1 ) == 1;\r\n\t\tif ( ics.TnsPresent ) ReadTns( r, ics );\r\n\r\n\t\tif ( r.Read( 1 ) == 1 ) throw new IndexOutOfRangeException( \"gain control\" ); // AAC SSR only\r\n\r\n\t\tReadSpectrum( r, ics );\r\n\t}\r\n\r\n\tvoid ReadSections( BitReader r, Ics ics )\r\n\t{\r\n\t\tvar bits = ics.IsLong ? 5 : 3;\r\n\t\tvar escape = (1 << bits) - 1;\r\n\t\tfor ( int g = 0; g < ics.NumWindowGroups; g++ )\r\n\t\t{\r\n\t\t\tvar k = 0;\r\n\t\t\twhile ( k < ics.MaxSfb )\r\n\t\t\t{\r\n\t\t\t\tvar cb = r.Read( 4 );\r\n\t\t\t\tif ( cb == 12 ) throw new IndexOutOfRangeException( \"reserved codebook\" );\r\n\t\t\t\tvar length = 0;\r\n\t\t\t\tint incr;\r\n\t\t\t\twhile ( (incr = r.Read( bits )) == escape ) length += escape;\r\n\t\t\t\tlength += incr;\r\n\t\t\t\tif ( k + length > ics.MaxSfb ) throw new IndexOutOfRangeException( \"section\" );\r\n\t\t\t\tfor ( int s = k; s < k + length; s++ ) ics.SfbCb[g * 64 + s] = cb;\r\n\t\t\t\tk += length;\r\n\t\t\t}\r\n\t\t}\r\n\t}\r\n\r\n\tvoid ReadScaleFactors( BitReader r, Ics ics )\r\n\t{\r\n\t\tvar scaleFactor = ics.GlobalGain;\r\n\t\tvar isPosition = 0;\r\n\t\tvar noiseEnergy = ics.GlobalGain - 90;\r\n\t\tvar noisePcm = true;\r\n\t\tfor ( int g = 0; g < ics.NumWindowGroups; g++ )\r\n\t\t{\r\n\t\t\tfor ( int sfb = 0; sfb < ics.MaxSfb; sfb++ )\r\n\t\t\t{\r\n\t\t\t\tvar i = g * 64 + sfb;\r\n\t\t\t\tswitch ( ics.SfbCb[i] )\r\n\t\t\t\t{\r\n\t\t\t\t\tcase ZeroHcb:\r\n\t\t\t\t\t\tics.ScaleFactors[i] = 0;\r\n\t\t\t\t\t\tbreak;\r\n\t\t\t\t\tcase IntensityHcb:\r\n\t\t\t\t\tcase IntensityHcb2:\r\n\t\t\t\t\t\tisPosition += Huffman.ScaleFactor.Decode( r ) - 60;\r\n\t\t\t\t\t\tics.ScaleFactors[i] = isPosition;\r\n\t\t\t\t\t\tbreak;\r\n\t\t\t\t\tcase NoiseHcb:\r\n\t\t\t\t\t\tif ( noisePcm ) { noisePcm = false; noiseEnergy += r.Read( 9 ) - 256; }\r\n\t\t\t\t\t\telse noiseEnergy += Huffman.ScaleFactor.Decode( r ) - 60;\r\n\t\t\t\t\t\tics.ScaleFactors[i] = noiseEnergy;\r\n\t\t\t\t\t\tbreak;\r\n\t\t\t\t\tdefault:\r\n\t\t\t\t\t\tscaleFactor += Huffman.ScaleFactor.Decode( r ) - 60;\r\n\t\t\t\t\t\tics.ScaleFactors[i] = scaleFactor;\r\n\t\t\t\t\t\tbreak;\r\n\t\t\t\t}\r\n\t\t\t}\r\n\t\t}\r\n\t}\r\n\r\n\tvoid ReadTns( BitReader r, Ics ics )\r\n\t{\r\n\t\tvar isLong = ics.IsLong;\r\n\t\tSpan<float> parcor = stackalloc float[32];\r\n\t\tSpan<float> a = stackalloc float[33];\r\n\t\tSpan<float> b = stackalloc float[33];\r\n\t\tfor ( int w = 0; w < ics.NumWindows; w++ )\r\n\t\t{\r\n\t\t\tvar filters = ics.TnsFilters[w] = r.Read( isLong ? 2 : 1 );\r\n\t\t\tif ( filters == 0 ) continue;\r\n\t\t\tvar coefRes = ics.TnsCoefRes[w] = r.Read( 1 );\r\n\t\t\tfor ( int f = 0; f < filters; f++ )\r\n\t\t\t{\r\n\t\t\t\tics.TnsLength[w, f] = r.Read( isLong ? 6 : 4 );\r\n\t\t\t\tvar order = ics.TnsOrder[w, f] = r.Read( isLong ? 5 : 3 );\r\n\t\t\t\tif ( order == 0 ) continue;\r\n\t\t\t\tics.TnsDirection[w, f] = r.Read( 1 );\r\n\t\t\t\tvar compress = r.Read( 1 );\r\n\t\t\t\tvar coefBits = coefRes + 3 - compress;\r\n\r\n\t\t\t\t// the coefficients are quantized reflection coefficients: back to a direct form LPC\r\n\t\t\t\tvar iqfac = ((1 << (coefRes + 3 - 1)) - 0.5f) / (MathF.PI / 2);\r\n\t\t\t\tvar iqfacM = ((1 << (coefRes + 3 - 1)) + 0.5f) / (MathF.PI / 2);\r\n\t\t\t\tfor ( int i = 0; i < order; i++ )\r\n\t\t\t\t{\r\n\t\t\t\t\tvar v = r.Read( coefBits );\r\n\t\t\t\t\tif ( (v & (1 << (coefBits - 1))) != 0 ) v -= 1 << coefBits;\r\n\t\t\t\t\tparcor[i] = MathF.Sin( v / (v >= 0 ? iqfac : iqfacM) );\r\n\t\t\t\t}\r\n\r\n\t\t\t\ta.Clear();\r\n\t\t\t\ta[0] = 1;\r\n\t\t\t\tfor ( int m = 1; m <= order; m++ )\r\n\t\t\t\t{\r\n\t\t\t\t\tfor ( int i = 1; i < m; i++ ) b[i] = a[i] + parcor[m - 1] * a[m - i];\r\n\t\t\t\t\tfor ( int i = 1; i < m; i++ ) a[i] = b[i];\r\n\t\t\t\t\ta[m] = parcor[m - 1];\r\n\t\t\t\t}\r\n\t\t\t\tfor ( int i = 0; i <= order; i++ ) ics.TnsLpc[w, f, i] = a[i];\r\n\t\t\t}\r\n\t\t}\r\n\t}\r\n\r\n\tvoid ReadSpectrum( BitReader r, Ics ics )\r\n\t{\r\n\t\tArray.Clear( ics.Quant );\r\n\t\tvar swb = ics.IsLong ? swbLong : swbShort;\r\n\t\tvar groupStart = 0; // first window of the group\r\n\r\n\t\tfor ( int g = 0; g < ics.NumWindowGroups; g++ )\r\n\t\t{\r\n\t\t\tvar groupLength = ics.WindowGroupLength[g];\r\n\t\t\tfor ( int sfb = 0; sfb < ics.MaxSfb; sfb++ )\r\n\t\t\t{\r\n\t\t\t\tvar cb = ics.SfbCb[g * 64 + sfb];\r\n\t\t\t\tif ( cb is ZeroHcb or NoiseHcb or IntensityHcb or IntensityHcb2 ) continue;\r\n\t\t\t\tvar book = Huffman.Spectral[cb];\r\n\t\t\t\tvar width = swb[sfb + 1] - swb[sfb];\r\n\r\n\t\t\t\t// within a group, a band's coefficients come window by window\r\n\t\t\t\tfor ( int w = 0; w < groupLength; w++ )\r\n\t\t\t\t{\r\n\t\t\t\t\tvar at = (groupStart + w) * 128 + swb[sfb];\r\n\t\t\t\t\tif ( ics.IsLong ) at = swb[sfb];\r\n\t\t\t\t\tfor ( int k = 0; k < width; k += book.Dimension )\r\n\t\t\t\t\t\tbook.DecodeValues( r, ics.Quant, at + k );\r\n\t\t\t\t}\r\n\t\t\t}\r\n\t\t\tgroupStart += groupLength;\r\n\t\t}\r\n\r\n\t\tif ( ics.PulsePresent )\r\n\t\t{\r\n\t\t\tint k = swbLong[Math.Min( ics.PulseStartSfb, numSwbLong )];\r\n\t\t\tfor ( int i = 0; i < ics.PulseCount; i++ )\r\n\t\t\t{\r\n\t\t\t\tk += ics.PulseOffset[i];\r\n\t\t\t\tif ( k >= 1024 ) break;\r\n\t\t\t\tics.Quant[k] += ics.Quant[k] > 0 ? ics.PulseAmp[i] : -ics.PulseAmp[i];\r\n\t\t\t}\r\n\t\t}\r\n\t}\r\n\r\n\tvoid SkipIcs( BitReader r )\r\n\t{\r\n\t\t// an extra channel we won't play still has to be read through\r\n\t\tDecodeIcs( r, icsB, false );\r\n\t}\r\n\r\n\tvoid DecodeCpe( BitReader r, int outputs )\r\n\t{\r\n\t\tvar a = icsA;\r\n\t\tvar b = icsB;\r\n\t\tvar commonWindow = r.Read( 1 ) == 1;\r\n\t\tvar msMask = 0;\r\n\t\tSpan<bool> msUsed = stackalloc bool[8 * 64];\r\n\t\tif ( commonWindow )\r\n\t\t{\r\n\t\t\tReadIcsInfo( r, a );\r\n\t\t\tCopyInfo( a, b );\r\n\t\t\tmsMask = r.Read( 2 );\r\n\t\t\tif ( msMask == 1 )\r\n\t\t\t\tfor ( int g = 0; g < a.NumWindowGroups; g++ )\r\n\t\t\t\t\tfor ( int sfb = 0; sfb < a.MaxSfb; sfb++ )\r\n\t\t\t\t\t\tmsUsed[g * 64 + sfb] = r.Read( 1 ) == 1;\r\n\t\t\telse if ( msMask == 2 ) msUsed.Fill( true );\r\n\t\t}\r\n\t\tDecodeIcs( r, a, commonWindow );\r\n\t\tDecodeIcs( r, b, commonWindow );\r\n\t\tif ( outputs + 1 >= 4 ) return;\r\n\r\n\t\tvar l = spectrum[0];\r\n\t\tvar rr = spectrum[1];\r\n\t\tDequantize( a, l );\r\n\t\tDequantize( b, rr );\r\n\r\n\t\t// PNS: a band that's noise in both channels with M/S on gets the same noise in both\r\n\t\tNoise( a, l, null, Span<bool>.Empty );\r\n\t\tNoise( b, rr, commonWindow && msMask > 0 ? a : null, commonWindow ? msUsed : Span<bool>.Empty, l );\r\n\r\n\t\tif ( commonWindow && msMask > 0 ) MidSide( a, b, l, rr, msUsed );\r\n\t\tIntensity( b, l, rr, msUsed, msMask );\r\n\r\n\t\tTns( a, l );\r\n\t\tTns( b, rr );\r\n\t\tSynthesize( a, l, outputs );\r\n\t\tSynthesize( b, rr, outputs + 1 );\r\n\t}\r\n\r\n\tstatic void CopyInfo( Ics from, Ics to )\r\n\t{\r\n\t\tto.WindowSequence = from.WindowSequence;\r\n\t\tto.WindowShape = from.WindowShape;\r\n\t\tto.MaxSfb = from.MaxSfb;\r\n\t\tto.NumWindows = from.NumWindows;\r\n\t\tto.NumWindowGroups = from.NumWindowGroups;\r\n\t\tArray.Copy( from.WindowGroupLength, to.WindowGroupLength, 8 );\r\n\t}\r\n\r\n\tvoid SkipPce( BitReader r )\r\n\t{\r\n\t\tr.Read( 4 + 2 + 4 ); // tag, object type, sampling index\r\n\t\tint front = r.Read( 4 ), side = r.Read( 4 ), back = r.Read( 4 ), lfe = r.Read( 2 ), assoc = r.Read( 3 ), cc = r.Read( 4 );\r\n\t\tif ( r.Read( 1 ) == 1 ) r.Read( 4 ); // mono mixdown\r\n\t\tif ( r.Read( 1 ) == 1 ) r.Read( 4 ); // stereo mixdown\r\n\t\tif ( r.Read( 1 ) == 1 ) r.Read( 3 ); // matrix mixdown\r\n\t\tr.Skip( (front + side + back) * 5 + lfe * 4 + assoc * 4 + cc * 5 );\r\n\t\tr.ByteAlign();\r\n\t\tr.Skip( r.Read( 8 ) * 8 ); // comment\r\n\t}\r\n\r\n\t//\r\n\t// spectral processing\r\n\t//\r\n\r\n\t/// <summary> Inverse quantization and scaling of one channel into <paramref name=\"spec\"/> (1024 values, window by window). </summary>\r\n\tvoid Dequantize( Ics ics, float[] spec )\r\n\t{\r\n\t\tArray.Clear( spec );\r\n\t\tvar swb = ics.IsLong ? swbLong : swbShort;\r\n\t\tvar window = 0;\r\n\t\tfor ( int g = 0; g < ics.NumWindowGroups; g++ )\r\n\t\t{\r\n\t\t\tfor ( int w = 0; w < ics.WindowGroupLength[g]; w++, window++ )\r\n\t\t\t{\r\n\t\t\t\tvar baseIndex = window * 128;\r\n\t\t\t\tfor ( int sfb = 0; sfb < ics.MaxSfb; sfb++ )\r\n\t\t\t\t{\r\n\t\t\t\t\tvar cb = ics.SfbCb[g * 64 + sfb];\r\n\t\t\t\t\tif ( cb is ZeroHcb or NoiseHcb or IntensityHcb or IntensityHcb2 ) continue;\r\n\t\t\t\t\tvar gain = MathF.Pow( 2, 0.25f * (ics.ScaleFactors[g * 64 + sfb] - 100) );\r\n\t\t\t\t\tfor ( int k = swb[sfb]; k < swb[sfb + 1]; k++ )\r\n\t\t\t\t\t{\r\n\t\t\t\t\t\tvar q = ics.Quant[baseIndex + k];\r\n\t\t\t\t\t\tif ( q != 0 ) spec[baseIndex + k] = (q > 0 ? Pow43( q ) : -Pow43( -q )) * gain;\r\n\t\t\t\t\t}\r\n\t\t\t\t}\r\n\t\t\t}\r\n\t\t}\r\n\t}\r\n\r\n\tstatic readonly float[] Pow43Table = MakePow43();\r\n\tstatic float[] MakePow43()\r\n\t{\r\n\t\tvar t = new float[8192];\r\n\t\tfor ( int i = 0; i < t.Length; i++ ) t[i] = MathF.Pow( i, 4.0f / 3.0f );\r\n\t\treturn t;\r\n\t}\r\n\tstatic float Pow43( int q ) => q < 8192 ? Pow43Table[q] : MathF.Pow( q, 4.0f / 3.0f );\r\n\r\n\t/// <summary> Only the single channel path: dequantize, noise, TNS. </summary>\r\n\tvoid Reconstruct( Ics ics, float[] spec )\r\n\t{\r\n\t\tDequantize( ics, spec );\r\n\t\tNoise( ics, spec, null, Span<bool>.Empty );\r\n\t\tTns( ics, spec );\r\n\t}\r\n\r\n\t/// <summary> Perceptual noise substitution: noise bands get random values of the coded energy. </summary>\r\n\tvoid Noise( Ics ics, float[] spec, Ics other, ReadOnlySpan<bool> msUsed, float[] otherSpec = null )\r\n\t{\r\n\t\tvar swb = ics.IsLong ? swbLong : swbShort;\r\n\t\tvar window = 0;\r\n\t\tfor ( int g = 0; g < ics.NumWindowGroups; g++ )\r\n\t\t{\r\n\t\t\tfor ( int w = 0; w < ics.WindowGroupLength[g]; w++, window++ )\r\n\t\t\t{\r\n\t\t\t\tvar baseIndex = window * 128;\r\n\t\t\t\tfor ( int sfb = 0; sfb < ics.MaxSfb; sfb++ )\r\n\t\t\t\t{\r\n\t\t\t\t\tvar i = g * 64 + sfb;\r\n\t\t\t\t\tif ( ics.SfbCb[i] != NoiseHcb ) continue;\r\n\t\t\t\t\tint from = baseIndex + swb[sfb], to = baseIndex + swb[sfb + 1];\r\n\r\n\t\t\t\t\tvar correlated = other is not null && otherSpec is not null && other.SfbCb[i] == NoiseHcb && msUsed.Length > i && msUsed[i];\r\n\t\t\t\t\tfloat energy = 0;\r\n\t\t\t\t\tfor ( int k = from; k < to; k++ )\r\n\t\t\t\t\t{\r\n\t\t\t\t\t\tvar v = correlated ? otherSpec[k] : NextNoise();\r\n\t\t\t\t\t\tspec[k] = v;\r\n\t\t\t\t\t\tenergy += v * v;\r\n\t\t\t\t\t}\r\n\t\t\t\t\tif ( energy <= 0 ) continue;\r\n\t\t\t\t\tvar scale = MathF.Pow( 2, 0.25f * ics.ScaleFactors[i] ) / MathF.Sqrt( energy );\r\n\t\t\t\t\tfor ( int k = from; k < to; k++ ) spec[k] *= scale;\r\n\t\t\t\t}\r\n\t\t\t}\r\n\t\t}\r\n\t}\r\n\r\n\tfloat NextNoise()\r\n\t{\r\n\t\tnoiseState = noiseState * 1664525u + 1013904223u;\r\n\t\treturn (int)noiseState / 2147483648.0f;\r\n\t}\r\n\r\n\tvoid MidSide( Ics ics, Ics right, float[] l, float[] r, ReadOnlySpan<bool> msUsed )\r\n\t{\r\n\t\tvar swb = ics.IsLong ? swbLong : swbShort;\r\n\t\tvar window = 0;\r\n\t\tfor ( int g = 0; g < ics.NumWindowGroups; g++ )\r\n\t\t{\r\n\t\t\tfor ( int w = 0; w < ics.WindowGroupLength[g]; w++, window++ )\r\n\t\t\t{\r\n\t\t\t\tfor ( int sfb = 0; sfb < ics.MaxSfb; sfb++ )\r\n\t\t\t\t{\r\n\t\t\t\t\tvar i = g * 64 + sfb;\r\n\t\t\t\t\tif ( !msUsed[i] || right.SfbCb[i] >= NoiseHcb || ics.SfbCb[i] == NoiseHcb ) continue;\r\n\t\t\t\t\tfor ( int k = window * 128 + swb[sfb]; k < window * 128 + swb[sfb + 1]; k++ )\r\n\t\t\t\t\t{\r\n\t\t\t\t\t\tvar m = l[k];\r\n\t\t\t\t\t\tvar s = r[k];\r\n\t\t\t\t\t\tl[k] = m + s;\r\n\t\t\t\t\t\tr[k] = m - s;\r\n\t\t\t\t\t}\r\n\t\t\t\t}\r\n\t\t\t}\r\n\t\t}\r\n\t}\r\n\r\n\tvoid Intensity( Ics right, float[] l, float[] r, ReadOnlySpan<bool> msUsed, int msMask )\r\n\t{\r\n\t\tvar swb = right.IsLong ? swbLong : swbShort;\r\n\t\tvar window = 0;\r\n\t\tfor ( int g = 0; g < right.NumWindowGroups; g++ )\r\n\t\t{\r\n\t\t\tfor ( int w = 0; w < right.WindowGroupLength[g]; w++, window++ )\r\n\t\t\t{\r\n\t\t\t\tfor ( int sfb = 0; sfb < right.MaxSfb; sfb++ )\r\n\t\t\t\t{\r\n\t\t\t\t\tvar i = g * 64 + sfb;\r\n\t\t\t\t\tvar cb = right.SfbCb[i];\r\n\t\t\t\t\tif ( cb is not (IntensityHcb or IntensityHcb2) ) continue;\r\n\t\t\t\t\tvar scale = MathF.Pow( 0.5f, 0.25f * right.ScaleFactors[i] );\r\n\t\t\t\t\tif ( cb == IntensityHcb2 ) scale = -scale;\r\n\t\t\t\t\tif ( msMask > 0 && msUsed[i] ) scale = -scale;\r\n\t\t\t\t\tfor ( int k = window * 128 + swb[sfb]; k < window * 128 + swb[sfb + 1]; k++ ) r[k] = l[k] * scale;\r\n\t\t\t\t}\r\n\t\t\t}\r\n\t\t}\r\n\t}\r\n\r\n\tvoid Tns( Ics ics, float[] spec )\r\n\t{\r\n\t\tif ( !ics.TnsPresent ) return;\r\n\t\tvar isLong = ics.IsLong;\r\n\t\tvar swb = isLong ? swbLong : swbShort;\r\n\t\tvar numSwb = isLong ? numSwbLong : numSwbShort;\r\n\t\tvar maxBands = Math.Min( (isLong ? AacTables.TnsMaxBandsLong : AacTables.TnsMaxBandsShort)[Math.Min( rateIndex, 11 )], ics.MaxSfb );\r\n\t\tSpan<float> state = stackalloc float[20];\r\n\r\n\t\tfor ( int w = 0; w < ics.NumWindows; w++ )\r\n\t\t{\r\n\t\t\tvar bottom = numSwb;\r\n\t\t\tfor ( int f = 0; f < ics.TnsFilters[w]; f++ )\r\n\t\t\t{\r\n\t\t\t\tvar top = bottom;\r\n\t\t\t\tbottom = Math.Max( top - ics.TnsLength[w, f], 0 );\r\n\t\t\t\tvar order = Math.Min( ics.TnsOrder[w, f], isLong ? 20 : 7 );\r\n\t\t\t\tif ( order == 0 ) continue;\r\n\r\n\t\t\t\tvar start = swb[Math.Min( bottom, maxBands )];\r\n\t\t\t\tvar end = swb[Math.Min( top, maxBands )];\r\n\t\t\t\tvar size = end - start;\r\n\t\t\t\tif ( size <= 0 ) continue;\r\n\r\n\t\t\t\tint at = w * 128 + (ics.TnsDirection[w, f] == 1 ? end - 1 : start);\r\n\t\t\t\tvar step = ics.TnsDirection[w, f] == 1 ? -1 : 1;\r\n\t\t\t\tstate.Clear();\r\n\t\t\t\tfor ( int n = 0; n < size; n++, at += step )\r\n\t\t\t\t{\r\n\t\t\t\t\t// all-pole filter: y = x - sum(a[i] * y[n-i])\r\n\t\t\t\t\tvar y = spec[at];\r\n\t\t\t\t\tfor ( int i = 0; i < order; i++ ) y -= state[i] * ics.TnsLpc[w, f, i + 1];\r\n\t\t\t\t\tfor ( int i = order - 1; i > 0; i-- ) state[i] = state[i - 1];\r\n\t\t\t\t\tstate[0] = y;\r\n\t\t\t\t\tspec[at] = y;\r\n\t\t\t\t}\r\n\t\t\t}\r\n\t\t}\r\n\t}\r\n\r\n\t//\r\n\t// filterbank\r\n\t//\r\n\r\n\tstatic readonly float[] SineLong = SineWindow( 2048 ), SineShort = SineWindow( 256 );\r\n\tstatic readonly float[] KbdLong = KbdWindow( 2048, 4 ), KbdShort = KbdWindow( 256, 6 );\r\n\r\n\tstatic float[] SineWindow( int n )\r\n\t{\r\n\t\tvar w = new float[n / 2];\r\n\t\tfor ( int i = 0; i < n / 2; i++ ) w[i] = MathF.Sin( MathF.PI / n * (i + 0.5f) );\r\n\t\treturn w;\r\n\t}\r\n\r\n\t/// <summary> The rising half of a Kaiser-Bessel-derived window. </summary>\r\n\tstatic float[] KbdWindow( int n, double alpha )\r\n\t{\r\n\t\tvar half = n / 2;\r\n\t\tvar kernel = new double[half + 1];\r\n\t\tfor ( int i = 0; i <= half; i++ )\r\n\t\t{\r\n\t\t\tvar x = (i - half / 2.0) / (half / 2.0);\r\n\t\t\tkernel[i] = BesselI0( Math.PI * alpha * Math.Sqrt( Math.Max( 0, 1 - x * x ) ) );\r\n\t\t}\r\n\t\tvar total = 0.0;\r\n\t\tfor ( int i = 0; i <= half; i++ ) total += kernel[i];\r\n\t\tvar w = new float[half];\r\n\t\tvar sum = 0.0;\r\n\t\tfor ( int i = 0; i < half; i++ )\r\n\t\t{\r\n\t\t\tsum += kernel[i];\r\n\t\t\tw[i] = (float)Math.Sqrt( sum / total );\r\n\t\t}\r\n\t\treturn w;\r\n\t}\r\n\r\n\tstatic double BesselI0( double x )\r\n\t{\r\n\t\tdouble sum = 1, term = 1;\r\n\t\tfor ( int k = 1; k < 50; k++ )\r\n\t\t{\r\n\t\t\tterm *= (x / (2 * k)) * (x / (2 * k));\r\n\t\t\tsum += term;\r\n\t\t\tif ( term < 1e-12 * sum ) break;\r\n\t\t}\r\n\t\treturn sum;\r\n\t}\r\n\r\n\tstatic float[] Rising( int shape, bool isLong ) => shape == 1 ? (isLong ? KbdLong : KbdShort) : (isLong ? SineLong : SineShort);\r\n\r\n\t/// <summary> IMDCT, window, overlap-add: 1024 new samples for output channel <paramref name=\"ch\"/>. </summary>\r\n\tvoid Synthesize( Ics ics, float[] spec, int ch )\r\n\t{\r\n\t\tif ( ch >= 4 ) return;\r\n\t\tvar t = timeBuffer;\r\n\t\tvar prevShape = previousShape[ch];\r\n\t\tvar shape = ics.WindowShape;\r\n\t\tvar longLeft = Rising( prevShape, true );\r\n\t\tvar longRight = Rising( shape, true );\r\n\t\tvar shortLeft = Rising( prevShape, false );\r\n\t\tvar shortRight = Rising( shape, false );\r\n\r\n\t\tif ( ics.WindowSequence == EightShort )\r\n\t\t{\r\n\t\t\tArray.Clear( t );\r\n\t\t\tSpan<float> block = stackalloc float[256];\r\n\t\t\tfor ( int w = 0; w < 8; w++ )\r\n\t\t\t{\r\n\t\t\t\tImdct.Short.Inverse( spec.AsSpan( w * 128, 128 ), block );\r\n\t\t\t\tvar left = w == 0 ? shortLeft : shortRight;\r\n\t\t\t\tvar at = 448 + w * 128;\r\n\t\t\t\tfor ( int i = 0; i < 128; i++ )\r\n\t\t\t\t{\r\n\t\t\t\t\tt[at + i] += block[i] * left[i];\r\n\t\t\t\t\tt[at + 128 + i] += block[128 + i] * shortRight[127 - i];\r\n\t\t\t\t}\r\n\t\t\t}\r\n\t\t}\r\n\t\telse\r\n\t\t{\r\n\t\t\tImdct.Long.Inverse( spec.AsSpan( 0, 1024 ), t );\r\n\t\t\tswitch ( ics.WindowSequence )\r\n\t\t\t{\r\n\t\t\t\tcase OnlyLong:\r\n\t\t\t\t\tfor ( int i = 0; i < 1024; i++ ) { t[i] *= longLeft[i]; t[1024 + i] *= longRight[1023 - i]; }\r\n\t\t\t\t\tbreak;\r\n\t\t\t\tcase LongStart:\r\n\t\t\t\t\tfor ( int i = 0; i < 1024; i++ ) t[i] *= longLeft[i];\r\n\t\t\t\t\tfor ( int i = 0; i < 128; i++ ) t[1472 + i] *= shortRight[127 - i];\r\n\t\t\t\t\tfor ( int i = 1600; i < 2048; i++ ) t[i] = 0;\r\n\t\t\t\t\tbreak;\r\n\t\t\t\tcase LongStop:\r\n\t\t\t\t\tfor ( int i = 0; i < 448; i++ ) t[i] = 0;\r\n\t\t\t\t\tfor ( int i = 0; i < 128; i++ ) t[448 + i] *= shortLeft[i];\r\n\t\t\t\t\tfor ( int i = 0; i < 1024; i++ ) t[1024 + i] *= longRight[1023 - i];\r\n\t\t\t\t\tbreak;\r\n\t\t\t}\r\n\t\t}\r\n\r\n\t\tvar o = overlap[ch];\r\n\t\tvar output = pcm[ch];\r\n\t\tfor ( int i = 0; i < 1024; i++ )\r\n\t\t{\r\n\t\t\toutput[i] = t[i] + o[i];\r\n\t\t\to[i] = t[1024 + i];\r\n\t\t}\r\n\t\tpreviousShape[ch] = shape;\r\n\t}\r\n\r\n\t/// <summary> Inverse MDCT through a DCT-IV of half the length, done with a complex FFT of a quarter. </summary>\r\n\tsealed class Imdct\r\n\t{\r\n\t\tpublic static readonly Imdct Long = new( 2048 ), Short = new( 256 );\r\n\r\n\t\treadonly int n, m; // window length, coefficient count (n / 2)\r\n\t\treadonly float[] twiddleCos, twiddleSin; // exp(-i pi (4k+1) / 4m)\r\n\t\treadonly Fft fft;\r\n\t\treadonly float[] re, im, dct;\r\n\r\n\t\tImdct( int length )\r\n\t\t{\r\n\t\t\tn = length;\r\n\t\t\tm = length / 2;\r\n\t\t\ttwiddleCos = new float[m / 2];\r\n\t\t\ttwiddleSin = new float[m / 2];\r\n\t\t\tfor ( int k = 0; k < m / 2; k++ )\r\n\t\t\t{\r\n\t\t\t\tvar a = -Math.PI * (4 * k + 1) / (4.0 * m);\r\n\t\t\t\ttwiddleCos[k] = (float)Math.Cos( a );\r\n\t\t\t\ttwiddleSin[k] = (float)Math.Sin( a );\r\n\t\t\t}\r\n\t\t\tfft = new Fft( m / 2 );\r\n\t\t\tre = new float[m / 2];\r\n\t\t\tim = new float[m / 2];\r\n\t\t\tdct = new float[m];\r\n\t\t}\r\n\r\n\t\t/// <summary> <paramref name=\"spec\"/>: m coefficients. <paramref name=\"output\"/>: n samples, scaled by 2/n. </summary>\r\n\t\tpublic void Inverse( ReadOnlySpan<float> spec, Span<float> output )\r\n\t\t{\r\n\t\t\tvar half = m / 2;\r\n\r\n\t\t\t// DCT-IV of spec into dct[]\r\n\t\t\tfor ( int k = 0; k < half; k++ )\r\n\t\t\t{\r\n\t\t\t\tfloat xr = spec[2 * k], xi = spec[m - 1 - 2 * k];\r\n\t\t\t\tre[k] = xr * twiddleCos[k] - xi * twiddleSin[k];\r\n\t\t\t\tim[k] = xr * twiddleSin[k] + xi * twiddleCos[k];\r\n\t\t\t}\r\n\t\t\tfft.Forward( re, im );\r\n\t\t\tfor ( int k = 0; k < half; k++ )\r\n\t\t\t{\r\n\t\t\t\tfloat yr = re[k] * twiddleCos[k] - im[k] * twiddleSin[k];\r\n\t\t\t\tfloat yi = re[k] * twiddleSin[k] + im[k] * twiddleCos[k];\r\n\t\t\t\tdct[2 * k] = yr;\r\n\t\t\t\tdct[m - 1 - 2 * k] = -yi;\r\n\t\t\t}\r\n\r\n\t\t\t// unfold to the IMDCT's n outputs (phase n0 = m/2 + 1/2)\r\n\t\t\tvar scale = 2.0f / n;\r\n\t\t\tfor ( int i = 0; i < half; i++ ) output[i] = dct[i + half] * scale;\r\n\t\t\tfor ( int i = half; i < 3 * half; i++ ) output[i] = -dct[3 * half - 1 - i] * scale;\r\n\t\t\tfor ( int i = 3 * half; i < n; i++ ) output[i] = -dct[i - 3 * half] * scale;\r\n\t\t}\r\n\t}\r\n\r\n\t/// <summary> In-place radix-2 complex FFT (forward, e^-i). </summary>\r\n\tsealed class Fft\r\n\t{\r\n\t\treadonly int size;\r\n\t\treadonly int[] reverse;\r\n\t\treadonly float[] cos, sin;\r\n\r\n\t\tpublic Fft( int size )\r\n\t\t{\r\n\t\t\tthis.size = size;\r\n\t\t\treverse = new int[size];\r\n\t\t\tvar bits = 0;\r\n\t\t\twhile ( (1 << bits) < size ) bits++;\r\n\t\t\tfor ( int i = 0; i < size; i++ )\r\n\t\t\t{\r\n\t\t\t\tvar r = 0;\r\n\t\t\t\tfor ( int b = 0; b < bits; b++ ) if ( (i & (1 << b)) != 0 ) r |= 1 << (bits - 1 - b);\r\n\t\t\t\treverse[i] = r;\r\n\t\t\t}\r\n\t\t\tcos = new float[size / 2];\r\n\t\t\tsin = new float[size / 2];\r\n\t\t\tfor ( int i = 0; i < size / 2; i++ )\r\n\t\t\t{\r\n\t\t\t\tcos[i] = (float)Math.Cos( -2 * Math.PI * i / size );\r\n\t\t\t\tsin[i] = (float)Math.Sin( -2 * Math.PI * i / size );\r\n\t\t\t}\r\n\t\t}\r\n\r\n\t\tpublic void Forward( float[] re, float[] im )\r\n\t\t{\r\n\t\t\tfor ( int i = 0; i < size; i++ )\r\n\t\t\t{\r\n\t\t\t\tvar j = reverse[i];\r\n\t\t\t\tif ( j <= i ) continue;\r\n\t\t\t\t(re[i], re[j]) = (re[j], re[i]);\r\n\t\t\t\t(im[i], im[j]) = (im[j], im[i]);\r\n\t\t\t}\r\n\t\t\tfor ( int len = 2; len <= size; len <<= 1 )\r\n\t\t\t{\r\n\t\t\t\tvar halfLen = len / 2;\r\n\t\t\t\tvar step = size / len;\r\n\t\t\t\tfor ( int i = 0; i < size; i += len )\r\n\t\t\t\t{\r\n\t\t\t\t\tfor ( int j = 0; j < halfLen; j++ )\r\n\t\t\t\t\t{\r\n\t\t\t\t\t\tfloat wr = cos[j * step], wi = sin[j * step];\r\n\t\t\t\t\t\tint a = i + j, b = a + halfLen;\r\n\t\t\t\t\t\tfloat tr = re[b] * wr - im[b] * wi;\r\n\t\t\t\t\t\tfloat ti = re[b] * wi + im[b] * wr;\r\n\t\t\t\t\t\tre[b] = re[a] - tr;\r\n\t\t\t\t\t\tim[b] = im[a] - ti;\r\n\t\t\t\t\t\tre[a] += tr;\r\n\t\t\t\t\t\tim[a] += ti;\r\n\t\t\t\t\t}\r\n\t\t\t\t}\r\n\t\t\t}\r\n\t\t}\r\n\t}\r\n\r\n\t//\r\n\t// Huffman\r\n\t//\r\n\r\n\t/// <summary> A codebook as a binary tree, built once from the code / length tables. </summary>\r\n\tsealed class Huffman\r\n\t{\r\n\t\tpublic static readonly Huffman ScaleFactor = new( AacTables.CodeScf, AacTables.LengthScf, 1, 0, false, false );\r\n\t\tpublic static readonly Huffman[] Spectral =\r\n\t\t{\r\n\t\t\tnull,\r\n\t\t\tnew( AacTables.Code1, AacTables.Length1, 4, 3, true, false ),\r\n\t\t\tnew( AacTables.Code2, AacTables.Length2, 4, 3, true, false ),\r\n\t\t\tnew( AacTables.Code3, AacTables.Length3, 4, 3, false, false ),\r\n\t\t\tnew( AacTables.Code4, AacTables.Length4, 4, 3, false, false ),\r\n\t\t\tnew( AacTables.Code5, AacTables.Length5, 2, 9, true, false ),\r\n\t\t\tnew( AacTables.Code6, AacTables.Length6, 2, 9, true, false ),\r\n\t\t\tnew( AacTables.Code7, AacTables.Length7, 2, 8, false, false ),\r\n\t\t\tnew( AacTables.Code8, AacTables.Length8, 2, 8, false, false ),\r\n\t\t\tnew( AacTables.Code9, AacTables.Length9, 2, 13, false, false ),\r\n\t\t\tnew( AacTables.Code10, AacTables.Length10, 2, 13, false, false ),\r\n\t\t\tnew( AacTables.Code11, AacTables.Length11, 2, 17, false, true ),\r\n\t\t};\r\n\r\n\t\tpublic readonly int Dimension;\r\n\t\treadonly int modulo;\r\n\t\treadonly bool signed, escape;\r\n\t\treadonly int[] tree; // node * 2 + bit: > 0 next node, <= 0 leaf -(index)\r\n\r\n\t\tHuffman( uint[] codes, byte[] lengths, int dimension, int modulo, bool signed, bool escape )\r\n\t\t{\r\n\t\t\tDimension = dimension;\r\n\t\t\tthis.modulo = modulo;\r\n\t\t\tthis.signed = signed;\r\n\t\t\tthis.escape = escape;\r\n\t\t\tvar nodes = new List<int> { 0, 0 };\r\n\t\t\tvar used = new List<bool> { false, false };\r\n\t\t\tfor ( int index = 0; index < codes.Length; index++ )\r\n\t\t\t{\r\n\t\t\t\tvar node = 0;\r\n\t\t\t\tfor ( int b = lengths[index] - 1; b >= 0; b-- )\r\n\t\t\t\t{\r\n\t\t\t\t\tvar bit = (int)(codes[index] >> b) & 1;\r\n\t\t\t\t\tvar slot = node * 2 + bit;\r\n\t\t\t\t\tif ( b == 0 )\r\n\t\t\t\t\t{\r\n\t\t\t\t\t\tnodes[slot] = -index;\r\n\t\t\t\t\t\tused[slot] = true;\r\n\t\t\t\t\t\tbreak;\r\n\t\t\t\t\t}\r\n\t\t\t\t\tif ( !used[slot] )\r\n\t\t\t\t\t{\r\n\t\t\t\t\t\tused[slot] = true;\r\n\t\t\t\t\t\tnodes[slot] = nodes.Count / 2;\r\n\t\t\t\t\t\tnodes.Add( 0 ); nodes.Add( 0 );\r\n\t\t\t\t\t\tused.Add( false ); used.Add( false );\r\n\t\t\t\t\t}\r\n\t\t\t\t\tnode = nodes[slot];\r\n\t\t\t\t}\r\n\t\t\t}\r\n\t\t\ttree = nodes.ToArray();\r\n\t\t}\r\n\r\n\t\tpublic int Decode( BitReader r )\r\n\t\t{\r\n\t\t\tvar node = 0;\r\n\t\t\twhile ( true )\r\n\t\t\t{\r\n\t\t\t\tvar next = tree[node * 2 + r.Read( 1 )];\r\n\t\t\t\tif ( next <= 0 ) return -next;\r\n\t\t\t\tnode = next;\r\n\t\t\t}\r\n\t\t}\r\n\r\n\t\t/// <summary> One codeword's values (4 or 2), with their signs and escapes, into <paramref name=\"dest\"/>. </summary>\r\n\t\tpublic void DecodeValues( BitReader r, int[] dest, int at )\r\n\t\t{\r\n\t\t\tvar index = Decode( r );\r\n\t\t\tSpan<int> v = stackalloc int[4];\r\n\t\t\tif ( Dimension == 4 )\r\n\t\t\t{\r\n\t\t\t\tv[0] = index / 27; v[1] = index / 9 % 3; v[2] = index / 3 % 3; v[3] = index % 3;\r\n\t\t\t}\r\n\t\t\telse\r\n\t\t\t{\r\n\t\t\t\tv[0] = index / modulo; v[1] = index % modulo;\r\n\t\t\t}\r\n\r\n\t\t\tif ( signed )\r\n\t\t\t{\r\n\t\t\t\tvar offset = Dimension == 4 ? 1 : 4;\r\n\t\t\t\tfor ( int i = 0; i < Dimension; i++ ) v[i] -= offset;\r\n\t\t\t}\r\n\t\t\telse\r\n\t\t\t{\r\n\t\t\t\tfor ( int i = 0; i < Dimension; i++ )\r\n\t\t\t\t\tif ( v[i] != 0 && r.Read( 1 ) == 1 ) v[i] = -v[i];\r\n\t\t\t}\r\n\r\n\t\t\tif ( escape )\r\n\t\t\t{\r\n\t\t\t\tfor ( int i = 0; i < 2; i++ )\r\n\t\t\t\t{\r\n\t\t\t\t\tif ( Math.Abs( v[i] ) != 16 ) continue;\r\n\t\t\t\t\tvar n = 4;\r\n\t\t\t\t\twhile ( r.Read( 1 ) == 1 ) { n++; if ( n > 12 ) throw new IndexOutOfRangeException( \"escape\" ); }\r\n\t\t\t\t\tvar value = (1 << n) + r.Read( n );\r\n\t\t\t\t\tv[i] = v[i] < 0 ? -value : value;\r\n\t\t\t\t}\r\n\t\t\t}\r\n\r\n\t\t\tfor ( int i = 0; i < Dimension && at + i < dest.Length; i++ ) dest[at + i] = v[i];\r\n\t\t}\r\n\t}\r\n\r\n\tsealed class BitReader\r\n\t{\r\n\t\treadonly byte[] data;\r\n\t\tint bit;\r\n\r\n\t\tpublic BitReader( byte[] data ) { this.data = data; }\r\n\r\n\t\tpublic int BitsLeft => data.Length * 8 - bit;\r\n\r\n\t\tpublic int Read( int n )\r\n\t\t{\r\n\t\t\tif ( n == 0 ) return 0;\r\n\t\t\tif ( bit + n > data.Length * 8 ) throw new IndexOutOfRangeException( \"end of frame\" );\r\n\t\t\tvar v = 0;\r\n\t\t\tfor ( int i = 0; i < n; i++, bit++ )\r\n\t\t\t\tv = (v << 1) | ((data[bit >> 3] >> (7 - (bit & 7))) & 1);\r\n\t\t\treturn v;\r\n\t\t}\r\n\r\n\t\tpublic void Skip( int n )\r\n\t\t{\r\n\t\t\tif ( bit + n > data.Length * 8 ) throw new IndexOutOfRangeException( \"end of frame\" );\r\n\t\t\tbit += n;\r\n\t\t}\r\n\r\n\t\tpublic void ByteAlign() => bit = (bit + 7) & ~7;\r\n\t}\r\n}\r\n"
},
{
"Ident": "rexi.bimp",
"Path": "Code/Resolver/Media/ISegmentFeed.cs",
"FileName": "ISegmentFeed.cs",
"PackageType": "library",
"CodeKind": "Game",
"AssetVersionId": 399292,
"IsPrivate": false,
"Code": "namespace Bimp.Resolver.Media;\r\n\r\n/// <summary>\r\n/// A source of finished segment files for <see cref=\"SegmentPlayer\"/>: merged YouTube media\r\n/// (<see cref=\"WebmSegmenter\"/>) or a live stream (<see cref=\"Live.LiveSegmenter\"/>).\r\n/// </summary>\r\npublic interface ISegmentFeed\r\n{\r\n\t/// <summary> Live: segments appear as the stream comes in, old ones go away, there's no end to seek in. </summary>\r\n\tbool IsLive { get; }\r\n\r\n\t/// <summary>\r\n\t/// Live: how much stream time must be buffered past the start point before playback starts, so the next\r\n\t/// segment is always finished before the current one ends. Sources that deliver in chunks (HLS) need more.\r\n\t/// </summary>\r\n\tdouble StartBufferSeconds => 0;\r\n\r\n\t/// <summary>\r\n\t/// Live, low latency: short segments that end in filler frames instead of an overlap - join the newest one and\r\n\t/// keep close behind the live edge.\r\n\t/// </summary>\r\n\tbool LowLatency => false;\r\n\r\n\t/// <summary> The newest finished segment, -1 if none yet. </summary>\r\n\tint Newest { get; }\r\n\r\n\t/// <summary> The segment if it's finished (and still on disk), else null. </summary>\r\n\tMediaSegment TryGet( int seq );\r\n\r\n\t/// <summary> Playback reached this segment - prepare it and what follows. </summary>\r\n\tvoid Want( int seq );\r\n\r\n\t/// <summary> Done with a segment - its file can go. </summary>\r\n\tvoid Release( int seq );\r\n\r\n\t/// <summary> Something went wrong, user facing. </summary>\r\n\tstring Error { get; }\r\n}\r\n"
},
{
"Ident": "rexi.bimp",
"Path": "Code/SegmentPlayer.cs",
"FileName": "SegmentPlayer.cs",
"PackageType": "library",
"CodeKind": "Game",
"AssetVersionId": 399292,
"IsPrivate": false,
"Code": "using Bimp.Resolver.Media;\n\nnamespace Bimp;\n\n/// <summary>\n/// Plays a series of local segment files back to back: merged YouTube media (<see cref=\"WebmSegmenter\"/>) or a\n/// live stream remuxed to MP4 (<see cref=\"Resolver.Live.LiveSegmenter\"/>).\n/// <para>\n/// Segment N plays while N+1 is started just in time in a second, hidden and silent player; at the boundary\n/// they swap, so the picture carries straight on. Segments keep absolute timestamps, so <see cref=\"Time\"/>\n/// is the media time.\n/// </para>\n/// </summary>\npublic sealed class SegmentPlayer : IDisposable\n{\n\tclass Slot\n\t{\n\t\tpublic MediaSegment Segment;\n\t\tpublic VideoPlayer Player;\n\t\tpublic bool Loaded;\n\t\tpublic bool Finished;\n\t\tpublic bool Preload; // waiting in the wings: silent until it's swapped in\n\t\tpublic bool FirstFrame; // set when it delivers its first frame (from OnTextureData, during Present)\n\t\tpublic bool Primed; // its decoder has produced its first frame\n\t\tpublic int Steady; // frames in a row it has delivered on time (see SettleFrames)\n\t\tpublic RealTimeSince SinceFrame;\n\t\tpublic bool Fading; // swapped out: silent (its volume ramps down) and closed a moment later\n\t\tpublic float LastTime;\n\t\tpublic RealTimeSince SinceOpened;\n\t}\n\n\treadonly ISegmentFeed feed;\n\n\t/// <summary>\n\t/// Live: start this many segments behind the newest finished one, so the next is always ready by the time the\n\t/// current one ends. Latency is roughly (JoinBehind + 1) segments plus the overlap.\n\t/// </summary>\n\tint JoinBehind => feed.LowLatency ? 0 : 1;\n\n\t/// <summary> Live: further behind than this (slow machine, stalled download) jumps back to the live edge. </summary>\n\tint MaxBehind => feed.LowLatency ? 1 : 3;\n\n\t/// <summary>\n\t/// Low latency: join the newest segment this long after it's written, so the next one is ready a little before\n\t/// the current one ends (its start latency plus arrival jitter). A segment that's late anyway just holds the\n\t/// last picture (the filler frames) until it arrives.\n\t/// </summary>\n\tconst double LowJoinMargin = 0.2;\n\n\t/// <summary>\n\t/// Average time from opening a segment to its first decoded frame (measured as we go).\n\t/// </summary>\n\tfloat startLatency;\n\n\t/// <summary>\n\t/// How long before the current segment ends to start the next one, so its first frame lands on the boundary -\n\t/// and so the two players are close to in step for the audio crossfade. A player that isn't primed in time\n\t/// just swaps a moment later (the current one plays on into its overlap, if the file has one).\n\t/// </summary>\n\tfloat StartLead => JustInTime\n\t\t? Math.Clamp( (startLatency > 0 ? startLatency : 0.08f) + 0.015f, 0.0f, 1.0f )\n\t\t: ClockLeadOnly ? Math.Clamp( clockLead, 0.0f, Math.Min( current.Segment.Overlap, 1.0f ) )\n\t\t: Math.Clamp( (startLatency > 0 ? startLatency : 0.04f) + leadCorrection, 0.0f, feed.IsLive ? 1.0f : 3.0f );\n\n\t/// <summary>\n\t/// Media files that run on into the next segment (<see cref=\"MediaSegment.Overlap\"/>): the current player shows real\n\t/// frames until the next one's first frame is up, however long that takes (150-520 ms at 1440p/4K), so the next one\n\t/// only has to be opened for its clock to be in step. Adding the start latency to that made it skip that far ahead\n\t/// at a swap, and correcting that back made the current one hold its last frame instead.\n\t/// <para>\n\t/// Merged WebM (VP9/AV1) only: MP4 segments (HLS VOD) kept waiting for the next H.264 player to settle made Media\n\t/// Foundation drop a sample (\"MFT deadlock\") in every run, and the picture broke up until the next keyframe (3 of 3\n\t/// runs, against 0 of 3 with the old timing).\n\t/// </para>\n\t/// </summary>\n\tbool ClockLeadOnly => feed is WebmSegmenter && (current?.Segment.Overlap ?? 0) > 0;\n\n\t/// <summary> How long before the end to open the next player so its clock is in step (<see cref=\"ClockLeadOnly\"/>), learned from each swap. </summary>\n\tfloat clockLead = 0.05f;\n\n\t/// <summary>\n\t/// <see cref=\"ClockLeadOnly\"/>: swap once the next player has delivered this many frames in a row on time. Its first\n\t/// frame comes late, and a 1440p decoder is still catching up to its clock for a moment after it - swapping on the\n\t/// first frame put that on screen as a 80-120 ms gap (measured). The current one plays on into its overlap meanwhile.\n\t/// </summary>\n\tconst int SettleFrames = 4;\n\tconst float SettleInterval = 0.045f;\n\n\t/// <summary>\n\t/// Low latency files that end at their boundary (H.264, ending in skip frames): the next one just opens early enough\n\t/// for its first frame to be ready there. (Syncing clocks made every swap ~40 ms late: a new player's clock runs from\n\t/// Play(), its first frame comes ~60 ms later.)\n\t/// </summary>\n\tbool JustInTime => feed.LowLatency && (current?.Segment.Overlap ?? 0) <= 0;\n\n\t/// <summary>\n\t/// Learned from each swap: how far the next player's clock was ahead of the current one's at the boundary.\n\t/// Starting it that much later next time brings them in step, which the audio crossfade needs - and so the next\n\t/// player neither skips its first frames nor makes the current one hold its last, while the current one plays on\n\t/// into its overlap (see <see cref=\"MediaSegment.Overlap\"/>).\n\t/// </summary>\n\tfloat leadCorrection;\n\n\t/// <summary>\n\t/// How long the swapped-out player keeps running, silent, after a swap.\n\t/// <para>\n\t/// The engine's mixer runs every sound through Steam Audio's direct effect, whose gain interpolator ramps any\n\t/// volume change over the next mix frames. Unmuting the next player while closing the current one outright\n\t/// left a ~70 ms dip in the audio at every boundary (measured with a sine: down to 5%, every segment).\n\t/// Muting the old player at the same moment instead - both ramps at once - makes that a crossfade.\n\t/// </para>\n\t/// </summary>\n\tconst float FadeSeconds = 0.15f;\n\n\tSlot fading;\n\tRealTimeSince sinceFade;\n\n\tSlot current, next;\n\tbool paused;\n\n\t// audio settings, applied to both players\n\tVector3 audioPosition;\n\tbool audioSpatial;\n\tfloat audioVolume = 1;\n\tfloat audioDistance = 2500;\n\n\tstring error;\n\tpublic string Error => error ?? feed.Error;\n\n\t/// <summary> The segment on screen, -1 before the first. </summary>\n\tpublic int CurrentSeq => current?.Segment.Seq ?? -1;\n\n\t/// <summary> Has the first segment started? </summary>\n\tpublic bool Loaded => current is { Loaded: true } || (Mjpeg?.HasShown ?? false);\n\n\t/// <summary> Motion JPEG: frames go straight to the texture, there are no segments to play. </summary>\n\tResolver.Live.LiveMjpeg Mjpeg => (feed as Resolver.Live.LiveSegmenter)?.Mjpeg;\n\n\t/// <summary> The last segment has played to its end. </summary>\n\tpublic bool Finished { get; private set; }\n\n\t/// <summary> Media time (VOD), or running stream time (live). </summary>\n\tpublic float Time => (float)StreamTime;\n\n\t/// <summary>\n\t/// Stream time (seconds) of what's on screen. WebM segments keep absolute timestamps (their StreamStart is 0); MP4\n\t/// segments (live, HLS VOD) each start their clock at 0.\n\t/// </summary>\n\tpublic double StreamTime => Mjpeg is { } mjpeg ? mjpeg.ShownTime : (current?.Segment.StreamStart ?? 0) + (current?.Player?.PlaybackTime ?? 0);\n\n\tpublic float Duration => current?.Player?.Duration ?? 0;\n\tpublic bool IsPaused => current?.Player?.IsPaused ?? paused;\n\n\t/// <summary> Every segment's frames go to one texture (see <see cref=\"VideoFrameSink\"/>). </summary>\n\treadonly VideoFrameSink sink = new();\n\n\tpublic Texture Texture => sink.Texture;\n\tpublic int Width => sink.Width;\n\tpublic int Height => sink.Height;\n\tpublic bool HasVideo => sink.Texture is not null && Width > 4 && Height > 4;\n\tpublic (int rate, int channels) AudioFormat => (current?.Player?.SampleRate ?? 0, current?.Player?.Channels ?? 0);\n\n\t/// <summary>\n\t/// Merged media: the synced media time, null while paused. A first segment that starts later than it (at the next\n\t/// keyframe - see <see cref=\"MediaBackend\"/>) opens only when the clock gets there, so it never has to seek forward.\n\t/// </summary>\n\tpublic float? SyncedTime { get; set; }\n\n\tpublic SegmentPlayer( ISegmentFeed feed )\n\t{\n\t\tthis.feed = feed;\n\t\tif ( !feed.IsLive ) feed.Want( 0 );\n\t}\n\n\t/// <summary>\n\t/// Call every frame.\n\t/// </summary>\n\tpublic void Update()\n\t{\n\t\tif ( Mjpeg is { } mjpeg )\n\t\t{\n\t\t\tif ( !paused ) mjpeg.Present( sink );\n\t\t\treturn;\n\t\t}\n\n\t\tif ( current is null )\n\t\t{\n\t\t\tvar first = feed.IsLive ? LiveJoinSegment() : feed.TryGet( 0 );\n\t\t\tif ( first is null ) return;\n\t\t\tif ( !feed.IsLive && !paused && SyncedTime is { } now && first.StreamStart + first.Start - now > StartLead ) return;\n\t\t\tcurrent = Open( first );\n\t\t\tif ( paused ) current.Player?.Pause();\n\t\t}\n\n\t\t// Live and fell far behind? Jump to the live edge.\n\t\tif ( feed.IsLive && feed.Newest - current.Segment.Seq > MaxBehind && LiveJoinSegment() is { } edge )\n\t\t{\n\t\t\tLog.Info( $\"[bimp] live: {feed.Newest - current.Segment.Seq} segments behind, jumping to the live edge\" );\n\t\t\tClose( next ); next = null;\n\t\t\tClose( current );\n\t\t\tcurrent = Open( edge );\n\t\t}\n\n\t\tTimedPresent( current, \"current\" );\n\n\t\t// the first segment's own start seeds the start latency, so the first swap isn't late on a slow (4K) decoder\n\t\tif ( startLatency <= 0 && !current.Primed && current.FirstFrame )\n\t\t{\n\t\t\tcurrent.Primed = true;\n\t\t\tstartLatency = (float)current.SinceOpened;\n\t\t}\n\n\t\t// the swapped-out player finishes its fade, then goes\n\t\tif ( fading is not null )\n\t\t{\n\t\t\tTimedPresent( fading, \"fading\" );\n\t\t\tif ( sinceFade > FadeSeconds ) CloseFading();\n\t\t}\n\n\t\tvar p = current.Player;\n\t\tvar seg = current.Segment;\n\n\t\t// A player started fresh delivers its first frame ~0.25s after Play(), and one that was paused takes ~1s\n\t\t// after Resume() - so start the next segment just in time (hidden, silent, presented so it runs),\n\t\t// timed from the measured start latency, and swap which player is on screen. Nothing ever pauses.\n\t\tif ( next is null && !paused && !seg.Last && current.Loaded && p is not null && p.PlaybackTime >= seg.End - StartLead )\n\t\t{\n\t\t\tvar upcoming = feed.TryGet( seg.Seq + 1 );\n\t\t\tif ( upcoming is not null ) next = Open( upcoming, preload: true );\n\t\t}\n\n\t\tif ( next?.Player is not null )\n\t\t{\n\t\t\tTimedPresent( next, \"next\" );\n\n\t\t\tif ( !next.Primed && next.FirstFrame )\n\t\t\t{\n\t\t\t\tnext.Primed = true;\n\t\t\t\tvar latency = (float)next.SinceOpened;\n\t\t\t\tstartLatency = startLatency <= 0 ? latency : startLatency * 0.7f + latency * 0.3f;\n\t\t\t\tMediaProbe.Note( $\"seg {next.Segment.Seq} first frame {latency * 1000:0}ms after opening (avg {startLatency * 1000:0}ms), current at {p?.PlaybackTime:0.000}/{seg.End:0.000}\" );\n\t\t\t}\n\t\t}\n\n\t\t// Swap when the current segment's clock reaches its end. (OnFinished fires ~80ms early, while\n\t\t// frames are still queued - swapping on it cut off the last frames.)\n\t\tvar stuck = current.Finished && p is not null && MathF.Abs( p.PlaybackTime - current.LastTime ) < 0.0001f;\n\t\tcurrent.LastTime = p?.PlaybackTime ?? 0;\n\t\tvar atEnd = (current.Loaded && p is not null && p.PlaybackTime >= seg.End - 0.005f) || stuck;\n\t\tif ( !atEnd ) return;\n\n\t\tif ( seg.Last )\n\t\t{\n\t\t\tFinished = current.Finished || stuck;\n\t\t\treturn;\n\t\t}\n\n\t\t// a next player still catching up waits, while the current one has overlap left to play\n\t\tvar settled = !ClockLeadOnly || next is null || next.Steady >= SettleFrames || (p is not null && p.PlaybackTime >= seg.End + seg.Overlap - 0.1f) || stuck;\n\t\tif ( next is { Primed: true } && settled )\n\t\t{\n\t\t\t// the next player's clock vs the current one's, in stream time: + means it's ahead. (Merged media keeps\n\t\t\t// absolute timestamps; live segments each start their clock at 0.)\n\t\t\tvar ahead = (float)((next.Segment.StreamStart + (next.Player?.PlaybackTime ?? 0)) - (seg.StreamStart + (p?.PlaybackTime ?? seg.End)));\n\n\t\t\t// keep the two clocks in step (see leadCorrection)\n\t\t\tif ( ClockLeadOnly ) clockLead = Math.Clamp( clockLead - ahead * 0.7f, 0.0f, 1.0f );\n\t\t\telse if ( !JustInTime ) leadCorrection = Math.Clamp( leadCorrection - ahead * 0.7f, -0.2f, 0.2f );\n\t\t\tMediaProbe.Note( $\"swap {seg.Seq}->{next.Segment.Seq} at {p?.PlaybackTime:0.000}/{seg.End:0.000} (overlap {seg.Overlap * 1000:0}ms), next ahead by {ahead * 1000:0}ms, lead now {StartLead * 1000:0}ms\" );\n\t\t\tnext.Preload = false;\n\t\t\tApplyAudio( next );\n\n\t\t\t// its frame from just before the boundary goes up now, not 33 ms later with its next one\n\t\t\tsink.ShowHeld( next.Player );\n\n\t\t\t// crossfade: the old player goes silent as the new one comes up (see FadeSeconds)\n\t\t\tCloseFading();\n\t\t\tfading = current;\n\t\t\tfading.Fading = true;\n\t\t\tApplyAudio( fading );\n\t\t\tsinceFade = 0;\n\n\t\t\tcurrent = next;\n\t\t\tnext = null;\n\t\t\tfeed.Want( current.Segment.Seq ); // keeps the one after it coming\n\t\t}\n\n\t\t// otherwise the next segment isn't downloaded yet - hold the last frame\n\t}\n\n\tvoid TimedPresent( Slot slot, string role )\n\t{\n\t\tif ( slot?.Player is null ) return;\n\t\tvar t = System.Diagnostics.Stopwatch.GetTimestamp();\n\t\tslot.Player.Present();\n\t\tMediaProbe.PresentTook( System.Diagnostics.Stopwatch.GetElapsedTime( t ).TotalMilliseconds, role, slot.Segment.Seq );\n\t}\n\n\t/// <summary>\n\t/// Live: the segment to start from - the newest one that's at least <see cref=\"JoinBehind\"/> behind the newest\n\t/// finished segment and leaves the feed's start buffer of stream time after it. Null while there isn't enough yet.\n\t/// </summary>\n\tMediaSegment LiveJoinSegment()\n\t{\n\t\tvar newest = feed.TryGet( feed.Newest );\n\t\tif ( newest is null ) return null;\n\t\tvar bufferedUntil = newest.StreamStart + newest.End;\n\n\t\tif ( feed.LowLatency ) return RealTime.NowDouble - newest.ReadyAt >= LowJoinMargin ? newest : null;\n\n\t\tfor ( var seq = newest.Seq - JoinBehind; seq >= 0; seq-- )\n\t\t{\n\t\t\tvar s = feed.TryGet( seq );\n\t\t\tif ( s is null ) return null; // not enough on disk yet\n\t\t\tif ( bufferedUntil - s.StreamStart >= feed.StartBufferSeconds ) return s;\n\t\t}\n\t\treturn null;\n\t}\n\n\tSlot Open( MediaSegment segment, bool preload = false )\n\t{\n\t\tMediaProbe.Note( $\"open seg {segment.Seq} ({segment.Start:0.00}-{segment.End:0.00}){(preload ? \" (preload)\" : \"\")}\" );\n\t\tvar slot = new Slot { Segment = segment, SinceOpened = 0, Preload = preload };\n\t\ttry\n\t\t{\n\t\t\tvar player = new VideoPlayer();\n\t\t\tplayer.OnLoaded += () => slot.Loaded = true;\n\t\t\tplayer.OnFinished += () => slot.Finished = true;\n\t\t\tslot.Player = player;\n\t\t\tApplyAudio( slot );\n\t\t\t// only the player on screen updates the texture; a preloading one just says it's ready\n\t\t\tsink.Attach( player, () => !slot.Preload && !slot.Fading, () =>\n\t\t\t{\n\t\t\t\tslot.Steady = slot.FirstFrame && slot.SinceFrame < SettleInterval ? slot.Steady + 1 : 0;\n\t\t\t\tslot.SinceFrame = 0;\n\t\t\t\tslot.FirstFrame = true;\n\t\t\t}, $\"seg {segment.Seq}\", holdWhileInactive: preload );\n\t\t\tvar playTimer = System.Diagnostics.Stopwatch.StartNew();\n\t\t\tplayer.Play( FileSystem.Data, segment.Path );\n\t\t\tMediaProbe.Note( $\"Play() seg {segment.Seq} took {playTimer.Elapsed.TotalMilliseconds:0.0}ms\" );\n\t\t}\n\t\tcatch ( Exception e )\n\t\t{\n\t\t\terror = e.Message;\n\t\t}\n\t\treturn slot;\n\t}\n\n\tvoid Close( Slot slot )\n\t{\n\t\tif ( slot?.Player is null ) return;\n\t\tvar timer = System.Diagnostics.Stopwatch.StartNew();\n\t\tsink.Detach( slot.Player );\n\t\tslot.Player.Dispose();\n\t\tMediaProbe.Note( $\"close seg {slot.Segment.Seq} took {timer.Elapsed.TotalMilliseconds:0.0}ms\" );\n\t}\n\n\tvoid CloseFading()\n\t{\n\t\tif ( fading is null ) return;\n\t\tClose( fading );\n\t\tfeed.Release( fading.Segment.Seq );\n\t\tfading = null;\n\t}\n\n\tpublic void SetPaused( bool pause )\n\t{\n\t\tpaused = pause;\n\t\tif ( current?.Player is { } p )\n\t\t{\n\t\t\tif ( pause ) p.Pause(); else p.Resume();\n\t\t}\n\n\t\tif ( pause ) CloseFading();\n\n\t\t// a preloading player would carry on running - start it again when we resume\n\t\tif ( pause && next is not null )\n\t\t{\n\t\t\tClose( next );\n\t\t\tnext = null;\n\t\t}\n\t}\n\n\t/// <summary>\n\t/// A short hop within the current segment (landing exactly on the synced time after opening).\n\t/// Anything further is done by opening a new segmenter at the time.\n\t/// </summary>\n\tpublic void Seek( float time )\n\t{\n\t\tvar seg = current?.Segment;\n\t\tif ( seg is null ) return;\n\t\tcurrent.Player?.Seek( Math.Clamp( time - (float)seg.StreamStart, seg.Start, seg.End ) );\n\t\tFinished = false;\n\t}\n\n\tpublic void SetAudio( Vector3 position, bool spatial, float volume, float distance )\n\t{\n\t\taudioPosition = position;\n\t\taudioSpatial = spatial;\n\t\taudioVolume = volume;\n\t\taudioDistance = distance;\n\t\tApplyAudio( current );\n\t\tApplyAudio( next );\n\t}\n\n\tvoid ApplyAudio( Slot slot )\n\t{\n\t\tif ( slot?.Player is null ) return;\n\t\tvar a = slot.Player.Audio;\n\t\ta.ListenLocal = !audioSpatial;\n\t\ta.Position = audioSpatial ? audioPosition : Vector3.Forward * 64.0f;\n\t\ta.Volume = slot.Preload || slot.Fading ? 0 : audioVolume;\n\t\ta.Distance = audioDistance;\n\t\ta.Falloff = MediaPlayer.AudioFalloff;\n\t}\n\n\tpublic void Dispose()\n\t{\n\t\tCloseFading();\n\t\tClose( current );\n\t\tClose( next );\n\t\tcurrent = next = null;\n\t\tsink.Dispose();\n\t}\n}\n"
},
{
"Ident": "rexi.bimp",
"Path": "Code/Resolver/Extractors/Extractors.cs",
"FileName": "Extractors.cs",
"PackageType": "library",
"CodeKind": "Game",
"AssetVersionId": 399292,
"IsPrivate": false,
"Code": "namespace Bimp.Resolver.Extractors;\n\n/// <summary>\n/// Every site the native resolver understands.\n/// </summary>\npublic static class ExtractorRegistry\n{\n\t// built on each call, not kept in a static: a hotload keeps statics, so new extractors never showed up (their caches\n\t// are statics of their own)\n\tpublic static IReadOnlyList<IExtractor> All => new IExtractor[]\n\t{\n\t\tnew YouTubeExtractor(),\n\t\tnew StreamableExtractor(),\n\t\tnew VimeoExtractor(),\n\t\tnew SoundCloudExtractor(),\n\t\tnew TwitchExtractor(),\n\t\tnew KickExtractor(),\n\t\tnew XExtractor(),\n\t\tnew ArchiveExtractor(),\n\t\tnew BandcampExtractor(),\n\t\tnew DailymotionExtractor(),\n\t\tnew HlsExtractor(), // any .m3u8 link\n\t\tnew DashExtractor(), // any .mpd link\n\t\tnew LabExtractor(), // [probe] loopback test media\n\t};\n\n\tpublic static IExtractor For( Uri url ) => All.FirstOrDefault( e => e.CanHandle( url ) );\n\n\tpublic static IExtractor ByKey( string key ) => All.FirstOrDefault( e => e.Key == key );\n}\n"
},
{
"Ident": "rexi.bimp",
"Path": "Code/MediaPlayer.cs",
"FileName": "MediaPlayer.cs",
"PackageType": "library",
"CodeKind": "Game",
"AssetVersionId": 399292,
"IsPrivate": false,
"Code": "using System;\r\nusing System.Collections.Generic;\r\nusing System.Linq;\r\nusing System.Threading.Tasks;\r\nusing Bimp.Resolver;\r\n\r\nnamespace Bimp;\r\n\r\n/// <summary>\r\n/// A networked media player. The host owns the state (what's playing, when it started, paused, queue)\r\n/// and every client plays it locally, keeping itself in sync with the host's clock.\r\n/// <para>\r\n/// Put this on a GameObject with a <see cref=\"WorldPanel\"/> + <see cref=\"MediaScreen\"/> for a TV/screen,\r\n/// or with <see cref=\"MediaSpeaker\"/> and <see cref=\"AudioOnlyPlayer\"/> for a speaker. Add a collider and\r\n/// <see cref=\"MediaInteract\"/> so players can open the controls with the use key.\r\n/// </para>\r\n/// </summary>\r\n[Title( \"Media Player\" ), Category( \"Media\" ), Icon( \"smart_display\" )]\r\npublic sealed class MediaPlayer : Component\r\n{\r\n\t//\r\n\t// Inspector settings\r\n\t//\r\n\r\n\t/// <summary>\r\n\t/// Who can control playback. The host can always control it.\r\n\t/// </summary>\r\n\t[Property, Group( \"Access\" )] public MediaPermission Permission { get; set; } = MediaPermission.Anyone;\r\n\r\n\t/// <summary>\r\n\t/// SteamIds allowed to control this player when <see cref=\"Permission\"/> is Whitelist.\r\n\t/// </summary>\r\n\t[Property, Group( \"Access\" ), ShowIf( nameof( Permission ), MediaPermission.Whitelist )]\r\n\tpublic List<long> AllowedSteamIds { get; set; } = new();\r\n\r\n\t/// <summary>\r\n\t/// Let anyone add to the queue, even if they can't control playback.\r\n\t/// </summary>\r\n\t[Property, Group( \"Access\" )] public bool AnyoneCanQueue { get; set; } = true;\r\n\r\n\t[Property, Group( \"Queue\" ), Range( 1, 200 )] public int MaxQueue { get; set; } = 30;\r\n\r\n\t/// <summary>\r\n\t/// Replay the current item when it ends and the queue is empty.\r\n\t/// </summary>\r\n\t[Property, Group( \"Queue\" )] public bool Loop { get; set; }\r\n\r\n\t/// <summary>\r\n\t/// Played by the host when the scene starts.\r\n\t/// </summary>\r\n\t[Property, Group( \"Playback\" )] public string DefaultUrl { get; set; }\r\n\r\n\t/// <summary>\r\n\t/// Never show video, only play audio. For speakers, radios and jukeboxes.\r\n\t/// </summary>\r\n\t[Property, Group( \"Playback\" )] public bool AudioOnlyPlayer { get; set; }\r\n\r\n\t[Property, Group( \"Playback\" ), Range( 0, 1 )] public float Volume { get; set; } = 1.0f;\r\n\r\n\t/// <summary>\r\n\t/// Sound comes from this object's position in the world and falls off with distance.\r\n\t/// When off, everyone hears it at the same volume wherever they are.\r\n\t/// </summary>\r\n\t[Property, Group( \"Audio\" )] public bool Spatial { get; set; } = true;\r\n\r\n\t/// <summary>\r\n\t/// Distance at which the sound is fully faded out (see <see cref=\"AudioFalloff\"/>).\r\n\t/// </summary>\r\n\t[Property, Group( \"Audio\" ), ShowIf( nameof( Spatial ), true )] public float AudioDistance { get; set; } = 4000.0f;\r\n\r\n\t/// <summary>\r\n\t/// How the sound fades out to <see cref=\"AudioDistance\"/>: nearly full volume in the room with the screen, then a\r\n\t/// smooth fade. The engine's own curve is made for its 15000 unit default range; over a screen's few thousand units\r\n\t/// it was down to 22% three metres away and 4% at twelve.\r\n\t/// </summary>\r\n\tpublic static readonly Curve AudioFalloff = new( new( 0, 1 ), new( 0.15f, 0.92f ), new( 0.45f, 0.55f ), new( 0.75f, 0.2f ), new( 1, 0 ) );\r\n\r\n\t/// <summary> 3D for this player: the player's own setting, unless they've turned 3D audio off (bimp_spatial). </summary>\r\n\tpublic bool EffectiveSpatial => Spatial && MediaSettings.SpatialAudio;\r\n\r\n\t/// <summary>\r\n\t/// 2D for this player (bimp_spatial off, on a player that's otherwise 3D): the sound has no direction, but still\r\n\t/// fades with the listener's distance from it, on the same curve as 3D. A player that isn't <see cref=\"Spatial\"/>\r\n\t/// at all is the same volume everywhere.\r\n\t/// </summary>\r\n\tbool FlatWithDistance => Spatial && !MediaSettings.SpatialAudio;\r\n\r\n\t/// <summary> How loud this player is at a point, from <see cref=\"AudioFalloff\"/> over <see cref=\"AudioDistance\"/>. </summary>\r\n\tfloat DistanceFade( Vector3 listener )\r\n\t{\r\n\t\tvar range = ScaledAudioDistance;\r\n\t\tif ( range <= 0 ) return 1;\r\n\t\treturn AudioFalloff.Evaluate( (listener.Distance( SoundPositionFor( listener ) ) / range).Clamp( 0, 1 ) );\r\n\t}\r\n\r\n\t/// <summary>\r\n\t/// Where the sound comes from. Defaults to this GameObject.\r\n\t/// </summary>\r\n\t[Property, Group( \"Audio\" )] public GameObject SoundOrigin { get; set; }\r\n\r\n\t//\r\n\t// Synced state - only the host writes these\r\n\t//\r\n\r\n\t/// <summary> The url the user entered. </summary>\r\n\t[Sync( SyncFlags.FromHost )] public string CurrentUrl { get; set; }\r\n\r\n\t/// <summary> The url clients actually play. Empty when nothing is playing. </summary>\r\n\t[Sync( SyncFlags.FromHost )] public string PlayUrl { get; set; }\r\n\r\n\t[Sync( SyncFlags.FromHost )] public string Title { get; set; }\r\n\t[Sync( SyncFlags.FromHost )] public string RequestedBy { get; set; }\r\n\r\n\t/// <summary> Media length in seconds, 0 if unknown or live. </summary>\r\n\t[Sync( SyncFlags.FromHost )] public float MediaDuration { get; set; }\r\n\r\n\t[Sync( SyncFlags.FromHost )] public bool IsLive { get; set; }\r\n\t[Sync( SyncFlags.FromHost )] public bool AudioOnly { get; set; }\r\n\t[Sync( SyncFlags.FromHost )] public bool SeekByReload { get; set; }\r\n\r\n\t/// <summary> Host time (<see cref=\"Time.NowDouble\"/>) at which media time 0 would have played. </summary>\r\n\t[Sync( SyncFlags.FromHost )] public double StartTime { get; set; }\r\n\r\n\t[Sync( SyncFlags.FromHost )] public bool Paused { get; set; }\r\n\t[Sync( SyncFlags.FromHost )] public float PausedAt { get; set; }\r\n\r\n\t/// <summary> Increments every time something (re)starts, so replaying the same url reloads it. </summary>\r\n\t[Sync( SyncFlags.FromHost )] public int PlayId { get; set; }\r\n\r\n\t/// <summary> Increments on every explicit seek, so clients apply it right away rather than treating it as drift. </summary>\r\n\t[Sync( SyncFlags.FromHost )] public int SeekId { get; set; }\r\n\r\n\t/// <summary> Host status message: \"Loading...\" or an error. </summary>\r\n\t[Sync( SyncFlags.FromHost )] public string Status { get; set; }\r\n\r\n\t[Sync( SyncFlags.FromHost )] public NetList<MediaQueueItem> Queue { get; set; } = new();\r\n\r\n\t/// <summary> Video heights the current media is available in (see <see cref=\"MediaQueueItem.Qualities\"/>). </summary>\r\n\t[Sync( SyncFlags.FromHost )] public string Qualities { get; set; }\r\n\r\n\t/// <summary> Audio tracks of the current media (see <see cref=\"MediaQueueItem.AudioTracks\"/>). </summary>\r\n\t[Sync( SyncFlags.FromHost )] public string AudioTracks { get; set; }\r\n\r\n\t/// <summary> Caption tracks of the current media (see <see cref=\"MediaQueueItem.CaptionTracks\"/>). </summary>\r\n\t[Sync( SyncFlags.FromHost )] public string CaptionTracks { get; set; }\r\n\r\n\t//\r\n\t// Per-client stream choices. Everyone shares the timeline, but picks their own resolution / audio language.\r\n\t//\r\n\r\n\tpublic IReadOnlyList<int> AvailableQualities => MediaStreamOptions.ParseQualities( Qualities );\r\n\tpublic IReadOnlyList<MediaAudioTrack> AvailableAudioTracks => MediaStreamOptions.ParseAudioTracks( AudioTracks );\r\n\r\n\t/// <summary> The height this client streams, 0 for automatic. From the bimp_quality convar. </summary>\r\n\tpublic int SelectedQuality => MediaStreamOptions.ChooseQuality( AvailableQualities, MediaSettings.Quality );\r\n\r\n\t/// <summary> The audio track this client streams, null for the original. From the bimp_audio_lang convar. </summary>\r\n\tpublic MediaAudioTrack SelectedAudioTrack => MediaStreamOptions.ChooseAudioTrack( AvailableAudioTracks, MediaSettings.AudioLanguage );\r\n\r\n\t/// <summary> <see cref=\"PlayUrl\"/> plus this client's choices. Changing a choice reloads playback at the same time. </summary>\r\n\tpublic string LocalStreamUrl => MediaStreamOptions.ApplyTo( PlayUrl, SelectedQuality, SelectedAudioTrack );\r\n\r\n\tstring CurrentKey => KeyFor( PlayId, LocalStreamUrl );\r\n\r\n\t//\r\n\t// Captions. Every client fetches its own cues (the urls can be tied to the IP) and shows them against the shared timeline.\r\n\t//\r\n\r\n\tpublic IReadOnlyList<MediaCaptionTrack> AvailableCaptionTracks => Captions.ParseTracks( CaptionTracks );\r\n\r\n\t/// <summary> The caption track this client shows, null when captions are off or there are none. From the bimp_subs / bimp_sub_lang convars. </summary>\r\n\tpublic MediaCaptionTrack SelectedCaptionTrack => MediaSettings.Subtitles ? Captions.Choose( AvailableCaptionTracks, MediaSettings.SubtitleLanguage ) : null;\r\n\r\n\t/// <summary> The caption line to show right now, null for none. </summary>\r\n\tpublic string CurrentCaption => HasMedia ? Captions.TextAt( captionCues, CurrentTime ) : null;\r\n\r\n\tstring captionKey;\r\n\tint captionToken;\r\n\tList<CaptionCue> captionCues;\r\n\r\n\t/// <summary> Fetch the cues when the media or the chosen track changes. </summary>\r\n\tvoid UpdateCaptions()\r\n\t{\r\n\t\tvar track = HasMedia ? SelectedCaptionTrack : null;\r\n\t\tvar key = track is null ? null : $\"{PlayId}|{track.Id}\";\r\n\t\tif ( key == captionKey ) return;\r\n\r\n\t\tcaptionKey = key;\r\n\t\tcaptionCues = null;\r\n\t\tvar token = ++captionToken;\r\n\t\tif ( track is not null ) LoadCaptionsAsync( token, track ).FireAndForget( \"captions\" );\r\n\t}\r\n\r\n\tasync Task LoadCaptionsAsync( int token, MediaCaptionTrack track )\r\n\t{\r\n\t\ttry\r\n\t\t{\r\n\t\t\tvar play = Bimp.Resolver.PlayToken.Parse( PlayUrl );\r\n\t\t\tif ( Bimp.Resolver.Extractors.ExtractorRegistry.ByKey( play?.Extractor ) is not Bimp.Resolver.ICaptionExtractor source ) return;\r\n\r\n\t\t\tvar body = await source.GetCaptionsAsync( play.Id, track.Id, System.Threading.CancellationToken.None );\r\n\t\t\tif ( token != captionToken || !this.IsValid() ) return;\r\n\r\n\t\t\tvar cues = Captions.Parse( body );\r\n\t\t\tLog.Info( $\"[bimp] captions {track.Id} ({track.Name}): {cues.Count} cues\" );\r\n\t\t\tcaptionCues = cues;\r\n\t\t}\r\n\t\tcatch ( Exception e )\r\n\t\t{\r\n\t\t\tLog.Warning( $\"[bimp] Couldn't load captions: {e.Message}\" );\r\n\t\t}\r\n\t}\r\n\r\n\t//\r\n\t// Local state\r\n\t//\r\n\r\n\t/// <summary>\r\n\t/// The local playback. Null on a dedicated server or when nothing is playing.\r\n\t/// </summary>\r\n\tpublic MediaBackend Backend { get; private set; }\r\n\r\n\tstring backendKey;\r\n\tRealTimeSince sinceCorrection;\r\n\tbool initialSeekPending;\r\n\tint reportedDurationFor = -1;\r\n\tint endedPlayId = -1;\r\n\tint resolveToken;\r\n\r\n\t/// <summary>\r\n\t/// Local message for this client (permission denied, load failures), fades after a few seconds.\r\n\t/// </summary>\r\n\tpublic string LocalNotice { get; private set; }\r\n\tRealTimeSince sinceNotice;\r\n\r\n\tpublic bool HasMedia => !string.IsNullOrEmpty( PlayUrl );\r\n\r\n\t/// <summary>\r\n\t/// Are we the authority on the state? True for the host, or when not networked at all.\r\n\t/// </summary>\r\n\tpublic bool IsAuthority => !Networking.IsActive || Networking.IsHost;\r\n\r\n\t/// <summary>\r\n\t/// Where playback should be right now, according to the host.\r\n\t/// </summary>\r\n\tpublic float CurrentTime\r\n\t{\r\n\t\tget\r\n\t\t{\r\n\t\t\tif ( !HasMedia ) return 0;\r\n\t\t\tvar t = Paused ? PausedAt : (float)Math.Max( 0, Time.NowDouble - StartTime );\r\n\t\t\tif ( MediaDuration > 0 ) t = MathF.Min( t, MediaDuration );\r\n\t\t\treturn t;\r\n\t\t}\r\n\t}\r\n\r\n\t/// <summary>\r\n\t/// The shared resolver (host only).\r\n\t/// </summary>\r\n\tstatic BimpResolver Resolver => BimpResolver.Instance;\r\n\r\n\tpublic Vector3 SoundPosition => SoundOrigin.IsValid() ? SoundOrigin.WorldPosition : WorldPosition;\r\n\r\n\tBoxCollider soundBox;\r\n\r\n\t/// <summary> The object the sound comes from: <see cref=\"SoundOrigin\"/>, or this one. </summary>\r\n\tGameObject SoundObject => SoundOrigin.IsValid() ? SoundOrigin : GameObject;\r\n\r\n\t/// <summary>\r\n\t/// How much bigger (or smaller) than its prefab this player has been scaled, e.g. by the Sandbox resize tool.\r\n\t/// The audio range grows with it, so a bigger screen is heard from further away.\r\n\t/// </summary>\r\n\tpublic float SoundScale\r\n\t{\r\n\t\tget\r\n\t\t{\r\n\t\t\tvar s = SoundObject.WorldScale;\r\n\t\t\treturn MathF.Max( 0.01f, MathF.Max( MathF.Abs( s.x ), MathF.Max( MathF.Abs( s.y ), MathF.Abs( s.z ) ) ) );\r\n\t\t}\r\n\t}\r\n\r\n\t/// <summary> <see cref=\"AudioDistance\"/> at the current size. </summary>\r\n\tpublic float ScaledAudioDistance => AudioDistance * SoundScale;\r\n\r\n\t/// <summary>\r\n\t/// Where the sound is heard from for a listener. A scaled-up screen is a big surface, not a point, so the sound comes\r\n\t/// from the nearest point of its box (the listener's own position when inside it), and fades from there.\r\n\t/// </summary>\r\n\tVector3 SoundPositionFor( Vector3 listener )\r\n\t{\r\n\t\tif ( SoundOrigin.IsValid() ) return SoundOrigin.WorldPosition;\r\n\r\n\t\tif ( !soundBox.IsValid() ) soundBox = GetComponent<BoxCollider>();\r\n\t\tif ( !soundBox.IsValid() ) return WorldPosition;\r\n\r\n\t\tvar tx = soundBox.GameObject.WorldTransform;\r\n\t\tvar half = soundBox.Scale * 0.5f;\r\n\t\tvar local = tx.PointToLocal( listener ) - soundBox.Center;\r\n\t\tlocal = new Vector3( local.x.Clamp( -half.x, half.x ), local.y.Clamp( -half.y, half.y ), local.z.Clamp( -half.z, half.z ) );\r\n\t\treturn tx.PointToWorld( local + soundBox.Center );\r\n\t}\r\n\r\n\tprotected override void OnAwake()\r\n\t{\r\n\t\t// Sync'd properties only work on network objects. Make that the default so mappers don't have to.\r\n\t\tif ( GameObject.NetworkMode == NetworkMode.Snapshot )\r\n\t\t\tGameObject.NetworkMode = NetworkMode.Object;\r\n\t}\r\n\r\n\tprotected override void OnStart()\r\n\t{\r\n\t\tif ( IsAuthority && !HasMedia && !string.IsNullOrWhiteSpace( DefaultUrl ) )\r\n\t\t{\r\n\t\t\tPlayNowAsync( DefaultUrl, \"Map\" ).FireAndForget( \"map autoplay\" );\r\n\t\t}\r\n\t}\r\n\r\n\tprotected override void OnDestroy()\r\n\t{\r\n\t\tDisposeBackend();\r\n\t}\r\n\r\n\tprotected override void OnDisabled()\r\n\t{\r\n\t\tDisposeBackend();\r\n\t}\r\n\r\n\tprotected override void OnUpdate()\r\n\t{\r\n\t\tif ( IsAuthority )\r\n\t\t\tUpdateAuthority();\r\n\r\n\t\tif ( !Application.IsDedicatedServer )\r\n\t\t{\r\n\t\t\tUpdatePlayback();\r\n\t\t\tUpdateCaptions();\r\n\t\t\tRecentLinks.Saw( this );\r\n\t\t}\r\n\r\n\t\tMediaProbe.PresentFile();\r\n\t\tLatencyLab.Tick();\r\n\t\tMediaProbe.SampleLatency( this );\r\n\t\tif ( MediaProbe.IsRecording( this ) )\r\n\t\t\tMediaProbe.Sample( this );\r\n\r\n\t\tif ( LocalNotice is not null && sinceNotice > 6 )\r\n\t\t\tLocalNotice = null;\r\n\t}\r\n\r\n\t#region Permissions\r\n\r\n\t/// <summary>\r\n\t/// Can this connection control playback?\r\n\t/// </summary>\r\n\tpublic bool CanControl( Connection c )\r\n\t{\r\n\t\tif ( !Networking.IsActive ) return true;\r\n\t\tif ( c is null ) return false;\r\n\t\tif ( c.IsHost || c == Connection.Host ) return true;\r\n\r\n\t\treturn Permission switch\r\n\t\t{\r\n\t\t\tMediaPermission.Anyone => true,\r\n\t\t\tMediaPermission.Whitelist => AllowedSteamIds?.Contains( c.SteamId.Value ) ?? false,\r\n\t\t\t_ => false,\r\n\t\t};\r\n\t}\r\n\r\n\t/// <summary>\r\n\t/// Can this connection add to the queue?\r\n\t/// </summary>\r\n\tpublic bool CanQueue( Connection c ) => AnyoneCanQueue || CanControl( c );\r\n\r\n\t/// <summary>\r\n\t/// Can the local player control playback? Used by the UI.\r\n\t/// </summary>\r\n\tpublic bool CanLocalControl => CanControl( Connection.Local );\r\n\tpublic bool CanLocalQueue => CanQueue( Connection.Local );\r\n\r\n\tstatic string CallerName( Connection c ) => c?.DisplayName ?? \"Someone\";\r\n\r\n\t#endregion\r\n\r\n\t#region Requests (called by anyone, run on the host)\r\n\r\n\t/// <summary>\r\n\t/// Play this url now, replacing whatever is playing.\r\n\t/// </summary>\r\n\t[Rpc.Host]\r\n\tpublic void RequestPlay( string url )\r\n\t{\r\n\t\tvar caller = Rpc.Caller;\r\n\t\tif ( !CanControl( caller ) ) { Deny( caller, \"You don't have permission to control this player.\" ); return; }\r\n\r\n\t\tPlayNowAsync( url, CallerName( caller ) ).FireAndForget( $\"play from {CallerName( caller )}\" );\r\n\t}\r\n\r\n\t/// <summary>\r\n\t/// Add this url to the queue. Plays immediately if nothing is playing.\r\n\t/// </summary>\r\n\t[Rpc.Host]\r\n\tpublic void RequestEnqueue( string url )\r\n\t{\r\n\t\tvar caller = Rpc.Caller;\r\n\t\tif ( !CanQueue( caller ) ) { Deny( caller, \"You don't have permission to queue on this player.\" ); return; }\r\n\t\tif ( Queue.Count >= MaxQueue ) { Deny( caller, \"The queue is full.\" ); return; }\r\n\r\n\t\tEnqueueAsync( url, CallerName( caller ), caller ).FireAndForget( $\"enqueue from {CallerName( caller )}\" );\r\n\t}\r\n\r\n\t[Rpc.Host]\r\n\tpublic void RequestSetPaused( bool paused )\r\n\t{\r\n\t\tif ( !CanControl( Rpc.Caller ) ) { Deny( Rpc.Caller, \"You don't have permission to control this player.\" ); return; }\r\n\t\tSetPaused( paused );\r\n\t}\r\n\r\n\t[Rpc.Host]\r\n\tpublic void RequestSeek( float time )\r\n\t{\r\n\t\tif ( !CanControl( Rpc.Caller ) ) { Deny( Rpc.Caller, \"You don't have permission to control this player.\" ); return; }\r\n\t\tSeek( time );\r\n\t}\r\n\r\n\t[Rpc.Host]\r\n\tpublic void RequestSkip()\r\n\t{\r\n\t\tif ( !CanControl( Rpc.Caller ) ) { Deny( Rpc.Caller, \"You don't have permission to control this player.\" ); return; }\r\n\t\tPlayNext();\r\n\t}\r\n\r\n\t[Rpc.Host]\r\n\tpublic void RequestStop()\r\n\t{\r\n\t\tif ( !CanControl( Rpc.Caller ) ) { Deny( Rpc.Caller, \"You don't have permission to control this player.\" ); return; }\r\n\t\tStop();\r\n\t}\r\n\r\n\t[Rpc.Host]\r\n\tpublic void RequestRemove( int index )\r\n\t{\r\n\t\tif ( !CanControl( Rpc.Caller ) ) { Deny( Rpc.Caller, \"You don't have permission to control this player.\" ); return; }\r\n\t\tif ( index < 0 || index >= Queue.Count ) return;\r\n\t\tQueue.RemoveAt( index );\r\n\t}\r\n\r\n\t[Rpc.Host]\r\n\tpublic void RequestClearQueue()\r\n\t{\r\n\t\tif ( !CanControl( Rpc.Caller ) ) { Deny( Rpc.Caller, \"You don't have permission to control this player.\" ); return; }\r\n\t\tQueue.Clear();\r\n\t}\r\n\r\n\t/// <summary>\r\n\t/// A client learned the duration of a direct file after loading it. The host takes the first answer.\r\n\t/// </summary>\r\n\t[Rpc.Host( NetFlags.Unreliable )]\r\n\tvoid ReportDuration( int playId, float duration )\r\n\t{\r\n\t\tif ( playId != PlayId || MediaDuration > 0 || IsLive ) return;\r\n\t\tif ( duration <= 0 || float.IsNaN( duration ) || float.IsInfinity( duration ) ) return;\r\n\t\tMediaDuration = duration;\r\n\t}\r\n\r\n\t/// <summary>\r\n\t/// Tell one client something went wrong with their request.\r\n\t/// </summary>\r\n\tvoid Deny( Connection caller, string message )\r\n\t{\r\n\t\tif ( caller is null || caller == Connection.Local || !Networking.IsActive )\r\n\t\t{\r\n\t\t\tShowNotice( message );\r\n\t\t\treturn;\r\n\t\t}\r\n\r\n\t\tusing ( Rpc.FilterInclude( caller ) )\r\n\t\t{\r\n\t\t\tReceiveNotice( message );\r\n\t\t}\r\n\t}\r\n\r\n\t[Rpc.Broadcast( NetFlags.HostOnly )]\r\n\tvoid ReceiveNotice( string message )\r\n\t{\r\n\t\tShowNotice( message );\r\n\t}\r\n\r\n\tvoid ShowNotice( string message )\r\n\t{\r\n\t\tLocalNotice = message;\r\n\t\tsinceNotice = 0;\r\n\t}\r\n\r\n\t#endregion\r\n\r\n\t#region Authority (host) logic\r\n\r\n\t/// <summary>\r\n\t/// Resolve and play right away. Host only.\r\n\t/// </summary>\r\n\tpublic async Task PlayNowAsync( string url, string requestedBy )\r\n\t{\r\n\t\tif ( !IsAuthority ) return;\r\n\r\n\t\tvar token = ++resolveToken;\r\n\t\tStatus = \"Loading...\";\r\n\r\n\t\ttry\r\n\t\t{\r\n\t\t\t// a playlist link: the first video plays now, the rest are queued\r\n\t\t\tvar playlist = await Resolver.ResolvePlaylistAsync( url, AudioOnlyPlayer, MaxQueue + 1 );\r\n\t\t\tif ( token != resolveToken || !this.IsValid() ) return;\r\n\t\t\tif ( playlist is not null )\r\n\t\t\t{\r\n\t\t\t\tvar first = await ResolveLazyAsync( playlist[0], requestedBy );\r\n\t\t\t\tif ( token != resolveToken || !this.IsValid() ) return;\r\n\r\n\t\t\t\tStartItem( first );\r\n\t\t\t\tQueueLazy( playlist.Skip( 1 ), requestedBy, null );\r\n\t\t\t\treturn;\r\n\t\t\t}\r\n\r\n\t\t\tvar item = await Resolver.ResolveAsync( url, AudioOnlyPlayer );\r\n\t\t\tif ( token != resolveToken || !this.IsValid() ) return; // something newer was requested\r\n\r\n\t\t\titem.RequestedBy = requestedBy;\r\n\t\t\tStartItem( item );\r\n\t\t}\r\n\t\tcatch ( Exception e )\r\n\t\t{\r\n\t\t\tif ( token != resolveToken || !this.IsValid() ) return;\r\n\t\t\tStatus = e is ResolveException ? e.Message : $\"Error: {e.Message}\";\r\n\t\t\tLog.Warning( $\"[bimp] Couldn't play {url}: {(e is ResolveException ? e.Message : e.ToString())}\" );\r\n\t\t}\r\n\t}\r\n\r\n\t/// <summary>\r\n\t/// Resolve and add to the queue. Host only.\r\n\t/// </summary>\r\n\tpublic async Task EnqueueAsync( string url, string requestedBy, Connection caller = null )\r\n\t{\r\n\t\tif ( !IsAuthority ) return;\r\n\r\n\t\ttry\r\n\t\t{\r\n\t\t\tvar playlist = await Resolver.ResolvePlaylistAsync( url, AudioOnlyPlayer, MaxQueue + 1 );\r\n\t\t\tif ( !this.IsValid() ) return;\r\n\t\t\tif ( playlist is not null )\r\n\t\t\t{\r\n\t\t\t\tvar rest = playlist.AsEnumerable();\r\n\t\t\t\tif ( !HasMedia && Status != \"Loading...\" )\r\n\t\t\t\t{\r\n\t\t\t\t\tvar first = await ResolveLazyAsync( playlist[0], requestedBy );\r\n\t\t\t\t\tif ( !this.IsValid() ) return;\r\n\t\t\t\t\tif ( !HasMedia && Status != \"Loading...\" ) // nothing else started meanwhile\r\n\t\t\t\t\t{\r\n\t\t\t\t\t\tStartItem( first );\r\n\t\t\t\t\t\trest = playlist.Skip( 1 );\r\n\t\t\t\t\t}\r\n\t\t\t\t}\r\n\r\n\t\t\t\tQueueLazy( rest, requestedBy, caller );\r\n\t\t\t\treturn;\r\n\t\t\t}\r\n\r\n\t\t\tvar item = await Resolver.ResolveAsync( url, AudioOnlyPlayer );\r\n\t\t\tif ( !this.IsValid() ) return;\r\n\r\n\t\t\titem.RequestedBy = requestedBy;\r\n\r\n\t\t\tif ( !HasMedia && Status != \"Loading...\" )\r\n\t\t\t{\r\n\t\t\t\tStartItem( item );\r\n\t\t\t\treturn;\r\n\t\t\t}\r\n\r\n\t\t\tif ( Queue.Count >= MaxQueue ) { Deny( caller, \"The queue is full.\" ); return; }\r\n\t\t\tQueue.Add( item );\r\n\t\t}\r\n\t\tcatch ( Exception e )\r\n\t\t{\r\n\t\t\tif ( !this.IsValid() ) return;\r\n\t\t\tDeny( caller, e is ResolveException ? e.Message : $\"Error: {e.Message}\" );\r\n\t\t}\r\n\t}\r\n\r\n\t/// <summary> A playlist entry resolved properly (qualities, dubs, live or not). Throws like <see cref=\"BimpResolver.ResolveAsync\"/>. </summary>\r\n\tasync Task<MediaQueueItem> ResolveLazyAsync( MediaQueueItem entry, string requestedBy )\r\n\t{\r\n\t\tvar item = await Resolver.ResolveAsync( entry.Url, AudioOnlyPlayer );\r\n\t\titem.RequestedBy = requestedBy;\r\n\t\treturn item;\r\n\t}\r\n\r\n\t/// <summary> Add playlist entries to the queue until it's full, and say so if some didn't fit. </summary>\r\n\tvoid QueueLazy( IEnumerable<MediaQueueItem> entries, string requestedBy, Connection caller )\r\n\t{\r\n\t\tint added = 0, left = 0;\r\n\t\tforeach ( var entry in entries )\r\n\t\t{\r\n\t\t\tif ( Queue.Count >= MaxQueue ) { left++; continue; }\r\n\t\t\tentry.RequestedBy = requestedBy;\r\n\t\t\tQueue.Add( entry );\r\n\t\t\tadded++;\r\n\t\t}\r\n\r\n\t\tif ( left > 0 ) Deny( caller, $\"Queued {added} videos - the queue is full, {left} more didn't fit.\" );\r\n\t}\r\n\r\n\t/// <summary> Play a queued playlist entry: resolve it first, and skip it (with a notice) if it can't be played. </summary>\r\n\tasync Task StartLazyAsync( MediaQueueItem entry )\r\n\t{\r\n\t\tvar token = resolveToken;\r\n\t\tStatus = \"Loading...\";\r\n\r\n\t\ttry\r\n\t\t{\r\n\t\t\tvar item = await ResolveLazyAsync( entry, entry.RequestedBy );\r\n\t\t\tif ( token != resolveToken || !this.IsValid() ) return;\r\n\t\t\tStartItem( item );\r\n\t\t}\r\n\t\tcatch ( Exception e )\r\n\t\t{\r\n\t\t\tif ( token != resolveToken || !this.IsValid() ) return;\r\n\r\n\t\t\tvar why = e is ResolveException ? e.Message : $\"Error: {e.Message}\";\r\n\t\t\tLog.Warning( $\"[bimp] Couldn't play {entry.Url}: {e.Message}\" );\r\n\r\n\t\t\tvar notice = $\"Skipped \\\"{entry.Title}\\\": {why}\";\r\n\t\t\tif ( Networking.IsActive ) ReceiveNotice( notice );\r\n\t\t\telse ShowNotice( notice );\r\n\r\n\t\t\tPlayNext();\r\n\t\t}\r\n\t}\r\n\r\n\tvoid StartItem( MediaQueueItem item )\r\n\t{\r\n\t\tCurrentUrl = item.Url;\r\n\t\tPlayUrl = item.PlayUrl;\r\n\t\tTitle = item.Title;\r\n\t\tRequestedBy = item.RequestedBy;\r\n\t\tMediaDuration = item.Duration;\r\n\t\tIsLive = item.IsLive;\r\n\t\tAudioOnly = item.AudioOnly || AudioOnlyPlayer;\r\n\t\tSeekByReload = item.SeekByReload;\r\n\t\tQualities = item.Qualities;\r\n\t\tAudioTracks = item.AudioTracks;\r\n\t\tCaptionTracks = item.CaptionTracks;\r\n\t\tPaused = false;\r\n\t\tPausedAt = 0;\r\n\t\t// a link with a start time (?t=90) plays from there, the way a late joiner starts mid-way\r\n\t\tStartTime = Time.NowDouble - (item.IsLive ? 0 : Math.Max( 0, item.StartAt ));\r\n\t\tStatus = null;\r\n\t\tPlayId++;\r\n\t}\r\n\r\n\t/// <summary>\r\n\t/// Play the next item in the queue, or stop. Host only.\r\n\t/// </summary>\r\n\tpublic void PlayNext()\r\n\t{\r\n\t\tif ( !IsAuthority ) return;\r\n\r\n\t\tresolveToken++; // cancel any pending \"play now\"\r\n\r\n\t\tif ( Queue.Count > 0 )\r\n\t\t{\r\n\t\t\tvar next = Queue[0];\r\n\t\t\tQueue.RemoveAt( 0 );\r\n\t\t\tif ( next.Lazy ) StartLazyAsync( next ).FireAndForget( $\"lazy start of {next.Title}\" );\r\n\t\t\telse StartItem( next );\r\n\t\t\treturn;\r\n\t\t}\r\n\r\n\t\tStop();\r\n\t}\r\n\r\n\t/// <summary>\r\n\t/// Stop playback (the queue is kept). Host only.\r\n\t/// </summary>\r\n\tpublic void Stop()\r\n\t{\r\n\t\tif ( !IsAuthority ) return;\r\n\r\n\t\tresolveToken++;\r\n\t\tCurrentUrl = null;\r\n\t\tPlayUrl = null;\r\n\t\tTitle = null;\r\n\t\tRequestedBy = null;\r\n\t\tQualities = null;\r\n\t\tAudioTracks = null;\r\n\t\tCaptionTracks = null;\r\n\t\tMediaDuration = 0;\r\n\t\tIsLive = false;\r\n\t\tPaused = false;\r\n\t\tPausedAt = 0;\r\n\t\tStatus = null;\r\n\t\tPlayId++;\r\n\t}\r\n\r\n\tpublic void SetPaused( bool paused )\r\n\t{\r\n\t\tif ( !IsAuthority || !HasMedia || IsLive || Paused == paused ) return;\r\n\r\n\t\tif ( paused )\r\n\t\t{\r\n\t\t\t// Freeze the shared timeline where the host's picture actually is, so the host's own video\r\n\t\t\t// doesn't jump on pause. Clients are within their drift tolerance of it anyway.\r\n\t\t\tPausedAt = LocalPlaybackPosition() ?? CurrentTime;\r\n\t\t\tPaused = true;\r\n\t\t}\r\n\t\telse\r\n\t\t{\r\n\t\t\tStartTime = Time.NowDouble - PausedAt;\r\n\t\t\tPaused = false;\r\n\t\t}\r\n\t}\r\n\r\n\tpublic void Seek( float time )\r\n\t{\r\n\t\tif ( !IsAuthority || !HasMedia || IsLive ) return;\r\n\r\n\t\ttime = MathF.Max( 0, time );\r\n\t\tif ( MediaDuration > 0 ) time = MathF.Min( time, MediaDuration - 0.5f );\r\n\r\n\t\tif ( Paused ) PausedAt = time;\r\n\t\telse StartTime = Time.NowDouble - time;\r\n\r\n\t\tSeekId++;\r\n\r\n\t\t// Don't let an old \"ended\" state trip us up after seeking back\r\n\t\tendedPlayId = -1;\r\n\t}\r\n\r\n\tvoid UpdateAuthority()\r\n\t{\r\n\t\tif ( !HasMedia || Paused || IsLive ) return;\r\n\t\tif ( endedPlayId == PlayId ) return;\r\n\r\n\t\tvar ended = false;\r\n\r\n\t\t// Known duration - use the clock, works on dedicated servers too\r\n\t\tif ( MediaDuration > 0 && Time.NowDouble - StartTime >= MediaDuration + 1.0 )\r\n\t\t\tended = true;\r\n\r\n\t\t// Unknown duration - trust our own playback, if we have one\r\n\t\tif ( MediaDuration <= 0 && Backend is not null && Backend.Finished && backendKey == CurrentKey )\r\n\t\t\tended = true;\r\n\r\n\t\tif ( !ended ) return;\r\n\r\n\t\tendedPlayId = PlayId;\r\n\t\tOnMediaEnded();\r\n\t}\r\n\r\n\tvoid OnMediaEnded()\r\n\t{\r\n\t\tif ( Queue.Count > 0 )\r\n\t\t{\r\n\t\t\tPlayNext();\r\n\t\t\treturn;\r\n\t\t}\r\n\r\n\t\tif ( Loop )\r\n\t\t{\r\n\t\t\tPaused = false;\r\n\t\t\tStartTime = Time.NowDouble;\r\n\t\t\tPlayId++; // force everyone to reload, the decoder has stopped\r\n\t\t\treturn;\r\n\t\t}\r\n\r\n\t\tStop();\r\n\t}\r\n\r\n\t#endregion\r\n\r\n\t#region Local playback\r\n\r\n\tstatic string KeyFor( int playId, string url ) => string.IsNullOrEmpty( url ) ? null : $\"{playId}|{url}\";\r\n\r\n\t/// <summary>\r\n\t/// Throw away local playback and start it again at the synced time.\r\n\t/// </summary>\r\n\tinternal void ReloadBackend()\r\n\t{\r\n\t\tDisposeBackend();\r\n\t\tbackendKey = CurrentKey;\r\n\t\tif ( backendKey is not null ) CreateBackend();\r\n\t}\r\n\r\n\t/// <summary>\r\n\t/// Diagnostics: open merged media at t with an explicit time offset.\r\n\t/// </summary>\r\n\tinternal void ReloadBackendAt( float t, float offset )\r\n\t{\r\n\t\tDisposeBackend();\r\n\t\tbackendKey = CurrentKey;\r\n\t\tif ( backendKey is null ) return;\r\n\t\tBackend = PlayToken.IsToken( LocalStreamUrl )\r\n\t\t\t? MediaBackend.CreateNative( LocalStreamUrl, AudioOnly, t )\r\n\t\t\t: MediaBackend.Create( LocalStreamUrl, AudioOnly, offset );\r\n\t\tsinceCorrection = 0;\r\n\t\tinitialSeekPending = false;\r\n\t\tappliedSeekId = SeekId;\r\n\t\tlastAppliedPaused = !Paused;\r\n\t}\r\n\r\n\tvoid DisposeBackend()\r\n\t{\r\n\t\tBackend?.Dispose();\r\n\t\tBackend = null;\r\n\t\tbackendKey = null;\r\n\t}\r\n\r\n\tvoid CreateBackend()\r\n\t{\r\n\t\t// Merged media starts at the synced time: segment files are written from the cue point before it,\r\n\t\t// with absolute timestamps, so there's no time offset. Everything else starts at 0 and seeks once loaded.\r\n\t\tvar t = CurrentTime;\r\n\t\tvar url = LocalStreamUrl;\r\n\r\n\t\tBackend = IsLive ? MediaBackend.CreateLive( url, Math.Clamp( MediaSettings.MaxVideoHeight, 144, 2160 ), AudioOnly )\r\n\t\t\t: PlayToken.IsToken( url ) ? MediaBackend.CreateNative( url, AudioOnly, SeekByReload && !IsLive ? t : 0 )\r\n\t\t\t: MediaBackend.Create( url, AudioOnly, 0 );\r\n\t\tsinceCorrection = 0;\r\n\t\tinitialSeekPending = !IsLive; // once loaded, land exactly on the synced time (see CorrectDrift)\r\n\t\tappliedSeekId = SeekId; // a fresh backend starts at the right time anyway (initial seek)\r\n\t\tlastAppliedPaused = !Paused; // make sure the new backend gets told the pause state\r\n\r\n\t\tif ( Backend.Error is not null )\r\n\t\t{\r\n\t\t\tnotifiedError = Backend;\r\n\t\t\tShowNotice( $\"Can't play this: {Backend.Error}\" );\r\n\t\t}\r\n\t}\r\n\r\n\tvoid UpdatePlayback()\r\n\t{\r\n\t\tvar wantKey = CurrentKey;\r\n\r\n\t\tif ( wantKey != backendKey )\r\n\t\t{\r\n\t\t\tDisposeBackend();\r\n\t\t\tbackendKey = wantKey;\r\n\r\n\t\t\tif ( wantKey is not null )\r\n\t\t\t\tCreateBackend();\r\n\t\t}\r\n\r\n\t\tif ( Backend is null ) return;\r\n\r\n\t\tBackend.SyncedTime = Paused || IsLive ? null : CurrentTime;\r\n\t\tBackend.Present();\r\n\t\tvar volume = Volume * MediaSettings.EffectiveVolume;\r\n\t\tif ( FlatWithDistance ) volume *= DistanceFade( Sound.Listener.Position );\r\n\t\tBackend.SetAudio( SoundPositionFor( Sound.Listener.Position ), EffectiveSpatial, volume, ScaledAudioDistance );\r\n\r\n\t\tif ( Backend.Error is not null )\r\n\t\t{\r\n\t\t\t// the stream's urls were refused (403/410) even after a refresh - re-resolve from the next client,\r\n\t\t\t// the same way a seek reopens, instead of sitting on the error\r\n\t\t\tif ( Backend.Refused && TryRefusalReload() )\r\n\t\t\t\treturn;\r\n\r\n\t\t\t// native media fails after the backend was created (resolving / downloading) - say so once\r\n\t\t\tif ( notifiedError != Backend )\r\n\t\t\t{\r\n\t\t\t\tnotifiedError = Backend;\r\n\t\t\t\tShowNotice( $\"Can't play this: {Backend.Error}\" );\r\n\t\t\t}\r\n\t\t\treturn;\r\n\t\t}\r\n\r\n\t\tif ( !Backend.Loaded )\r\n\t\t{\r\n\t\t\tif ( Backend.SinceCreated > 20 && LocalNotice is null )\r\n\t\t\t\tShowNotice( \"This is taking a while to load...\" );\r\n\t\t\treturn;\r\n\t\t}\r\n\r\n\t\t// Tell the host the duration of direct files, it can't know until something loads them\r\n\t\tif ( MediaDuration <= 0 && !IsLive && !SeekByReload && reportedDurationFor != PlayId && Backend.Duration > 0 )\r\n\t\t{\r\n\t\t\treportedDurationFor = PlayId;\r\n\t\t\tReportDuration( PlayId, Backend.Duration );\r\n\t\t}\r\n\r\n\t\t// Pausing/resuming gets a moment to settle (the decoder refills its buffers on resume) before\r\n\t\t// anything is judged as drift or a stall - correcting then is what caused stutter after unpausing\r\n\t\t// A freshly opened player (after a seek, join or reload) runs until it has landed on the synced time,\r\n\t\t// even when paused - otherwise a seek while paused leaves the screen without the new frame\r\n\t\tvar wantPaused = Paused && !initialSeekPending;\r\n\t\tif ( wantPaused != lastAppliedPaused )\r\n\t\t{\r\n\t\t\tlastAppliedPaused = wantPaused;\r\n\t\t\tsinceCorrection = 0;\r\n\t\t\tResetStallTimer();\r\n\t\t}\r\n\r\n\t\tBackend.SetPaused( wantPaused );\r\n\r\n\t\t// A stuck live stream (source hiccup, dropped connection) reconnects at the live edge\r\n\t\tif ( CheckStalled() ) return;\r\n\r\n\t\tif ( IsLive ) return; // nothing to sync to - everyone watches the live edge\r\n\r\n\t\tCorrectDrift();\r\n\t}\r\n\r\n\tbool lastAppliedPaused;\r\n\tint appliedSeekId;\r\n\tMediaBackend notifiedError;\r\n\r\n\t/// <summary>\r\n\t/// Where this client's playback actually is, if it's loaded and roughly in sync (null otherwise).\r\n\t/// </summary>\r\n\tfloat? LocalPlaybackPosition()\r\n\t{\r\n\t\tif ( Backend is null || Backend.Error is not null || !Backend.Loaded || backendKey != CurrentKey ) return null;\r\n\r\n\t\tvar t = Backend.Time;\r\n\t\tif ( MathF.Abs( t - CurrentTime ) > 3.0f ) return null; // way off (stalled / still seeking) - don't trust it\r\n\t\tif ( MediaDuration > 0 ) t = MathF.Min( t, MediaDuration );\r\n\t\treturn MathF.Max( 0, t );\r\n\t}\r\n\r\n\tconst float StallSeconds = 5.0f;\r\n\tconst int MaxStallReloads = 3;\r\n\r\n\tfloat lastProgressTime;\r\n\tRealTimeSince sinceProgress;\r\n\tint stallReloads;\r\n\tint stallPlayId = -1;\r\n\r\n\t/// <summary>\r\n\t/// If playback should be moving but hasn't for a while (the decoder got stuck after a seek,\r\n\t/// the connection dropped...), throw the backend away and start again at the right time.\r\n\t/// Returns true if it reloaded.\r\n\t/// </summary>\r\n\tbool CheckStalled()\r\n\t{\r\n\t\tif ( stallPlayId != PlayId )\r\n\t\t{\r\n\t\t\tstallPlayId = PlayId;\r\n\t\t\tstallReloads = 0;\r\n\t\t\tResetStallTimer();\r\n\t\t}\r\n\r\n\t\tvar t = Backend.Time;\r\n\t\tif ( Paused || MathF.Abs( t - lastProgressTime ) > 0.05f )\r\n\t\t{\r\n\t\t\tlastProgressTime = t;\r\n\t\t\tsinceProgress = 0;\r\n\t\t\treturn false;\r\n\t\t}\r\n\r\n\t\t// Reached the end, not stuck\r\n\t\tif ( MediaDuration > 0 && t >= MediaDuration - 1.0f ) return false;\r\n\r\n\t\tif ( sinceProgress < StallSeconds || stallReloads >= MaxStallReloads ) return false;\r\n\r\n\t\tstallReloads++;\r\n\t\tLog.Warning( $\"[bimp] Playback stalled at {t:0.0}s, reloading ({stallReloads}/{MaxStallReloads})\" );\r\n\t\tShowNotice( \"Playback got stuck, reloading...\" );\r\n\r\n\t\tDisposeBackend();\r\n\t\tbackendKey = CurrentKey;\r\n\t\tCreateBackend();\r\n\t\tResetStallTimer();\r\n\t\treturn true;\r\n\t}\r\n\r\n\tvoid ResetStallTimer()\r\n\t{\r\n\t\tlastProgressTime = -1;\r\n\t\tsinceProgress = 0;\r\n\t}\r\n\r\n\tconst int MaxRefusalReloads = 2;\r\n\tint refusalReloads;\r\n\tint refusalPlayId = -1;\r\n\r\n\t/// <summary>\r\n\t/// The stream's urls were refused even after a refresh: forget the cached session and reload, so it resolves\r\n\t/// fresh from the next source (StreamSessions skips the client that just died). Budgeted like a stall.\r\n\t/// </summary>\r\n\tbool TryRefusalReload()\r\n\t{\r\n\t\tif ( refusalPlayId != PlayId ) { refusalPlayId = PlayId; refusalReloads = 0; }\r\n\t\tif ( refusalReloads >= MaxRefusalReloads ) return false;\r\n\r\n\t\trefusalReloads++;\r\n\t\tLog.Warning( $\"[bimp] stream refused at {Backend.Time:0.0}s, re-resolving from another source ({refusalReloads}/{MaxRefusalReloads})\" );\r\n\t\tShowNotice( \"The stream stopped answering, switching source...\" );\r\n\r\n\t\tBackend.InvalidateSession();\r\n\t\tReloadBackend();\r\n\t\tResetStallTimer();\r\n\t\treturn true;\r\n\t}\r\n\r\n\t/// <summary> Merged media lands within this of the synced time without seeking (the drift threshold). </summary>\r\n\tconst float MergedLandingTolerance = 1.5f;\r\n\r\n\t/// <summary> The play and seek that merged media last reopened for, after landing too far behind. </summary>\r\n\t(int playId, int seekId) landingReloadFor = (-1, -1);\r\n\r\n\t/// <summary>\r\n\t/// Keep local playback close to where the host says it should be.\r\n\t/// </summary>\r\n\tvoid CorrectDrift()\r\n\t{\r\n\t\tvar expected = CurrentTime;\r\n\t\tvar drift = Backend.Time - expected;\r\n\t\tvar absDrift = MathF.Abs( drift );\r\n\r\n\t\t// Just loaded: land exactly on the synced time. For merged media the first segment starts at the keyframe\r\n\t\t// cluster before it, so this is a short hop forward inside the first cluster (a few seconds at most),\r\n\t\t// which the engine's player gets through in a blink.\r\n\t\tif ( initialSeekPending )\r\n\t\t{\r\n\t\t\t// The loaded event can come a frame or two before the player's clock reports the segment's position\r\n\t\t\t// (it reads 0) - wait for it, or we'd skip the hop and sit a second behind\r\n\t\t\tif ( SeekByReload && absDrift >= 15.0f && Backend.SinceCreated < 10.0f ) return;\r\n\r\n\t\t\tinitialSeekPending = false;\r\n\t\t\tappliedSeekId = SeekId;\r\n\t\t\t// Merged media started at a keyframe just behind the synced time (or waited for the next one): it keeps that\r\n\t\t\t// small lag rather than seeking forward, which means decoding everything in between while the sound runs\r\n\t\t\t// on - at 4K the picture then crawled behind for seconds (see MediaBackend.StartNative).\r\n\t\t\tMediaProbe.Note( $\"initial landing: drift {drift * 1000:0}ms\" );\r\n\t\t\tvar tolerance = SeekByReload ? MergedLandingTolerance : 0.25f;\r\n\t\t\tif ( absDrift > tolerance && (!SeekByReload || absDrift < 15.0f) )\r\n\t\t\t{\r\n\t\t\t\t// Landed further behind than planned (a slow load): open again, now planned with the real load time,\r\n\t\t\t\t// rather than seek forward. Once per play or seek, so a load that's always slow can't loop.\r\n\t\t\t\tif ( SeekByReload && drift < 0 && landingReloadFor != (PlayId, SeekId) )\r\n\t\t\t\t{\r\n\t\t\t\t\tlandingReloadFor = (PlayId, SeekId);\r\n\t\t\t\t\tMediaProbe.Note( \"landed too far behind: reopening\" );\r\n\t\t\t\t\tReloadBackend();\r\n\t\t\t\t\treturn;\r\n\t\t\t\t}\r\n\r\n\t\t\t\tBackend.Seek( expected );\r\n\t\t\t\tsinceCorrection = 0;\r\n\t\t\t}\r\n\t\t\treturn;\r\n\t\t}\r\n\r\n\t\t// Someone seeked (button / progress bar): apply it now, not as drift after a cooldown\r\n\t\tvar explicitSeek = appliedSeekId != SeekId;\r\n\t\tappliedSeekId = SeekId;\r\n\r\n\t\t// The same tolerance applies while paused - pausing must never seek (our playback normally trails\r\n\t\t// the synced clock by up to a second of load latency; correcting that on pause made it jump).\r\n\t\tconst float threshold = 1.5f;\r\n\t\tvar drifted = absDrift >= threshold && sinceCorrection >= 3.0f;\r\n\t\tif ( !(explicitSeek && absDrift > 0.25f) && !drifted ) return;\r\n\r\n\t\tif ( SeekByReload )\r\n\t\t{\r\n\t\t\t// ROOT CAUSE of \"it keeps skipping forward frames\": the engine's player can't seek forward in a\r\n\t\t\t// stream. Seek() jumps its clock but it keeps reading and decoding sequentially from where it was,\r\n\t\t\t// throwing away every frame as late until it catches up - fast forwarding for as long as the gap\r\n\t\t\t// was (minutes, for a big seek), while drift correction kept firing more seeks. So merged\r\n\t\t\t// media never asks it to jump: it reopens at the target, writing segment files that start there.\r\n\t\t\tif ( MediaDuration > 0 && expected >= MediaDuration - 2 ) return;\r\n\t\t\tReloadBackend();\r\n\t\t\treturn;\r\n\t\t}\r\n\r\n\t\t// Direct files: the native seek (fine for backward seeks and short hops)\r\n\t\tBackend.Seek( expected );\r\n\t\tsinceCorrection = 0;\r\n\t}\r\n\r\n\t#endregion\r\n}\r\n"
},
{
"Ident": "rexi.bimp",
"Path": "MediaBackend.cs",
"FileName": "MediaBackend.cs",
"PackageType": "library",
"CodeKind": "Game",
"AssetVersionId": 399292,
"IsPrivate": false,
"Code": "using System;\r\nusing Bimp.Resolver;\r\nusing Bimp.Resolver.Media;\r\n\r\nnamespace Bimp;\r\n\r\n/// <summary>\r\n/// Local playback of a single url or play token. Wraps the engine's <see cref=\"VideoPlayer\"/> (video),\r\n/// <see cref=\"MusicPlayer\"/> (audio only, gives us a spectrum) or a <see cref=\"SegmentPlayer\"/> (merged\r\n/// YouTube media) behind one interface.\r\n/// Each client owns one of these per <see cref=\"MediaPlayer\"/>, it is never networked.\r\n/// </summary>\r\npublic sealed class MediaBackend : IDisposable\r\n{\r\n\tVideoPlayer video;\r\n\tMusicPlayer music;\r\n\r\n\t/// <summary> The direct video player's frames (see <see cref=\"VideoFrameSink\"/>). </summary>\r\n\treadonly VideoFrameSink videoSink = new();\r\n\tSegmentPlayer segments;\r\n\tWebmSegmenter segmenter;\r\n\tResolver.Live.LiveStream live;\r\n\r\n\t/// <summary>\r\n\t/// Natively resolved media being prepared (resolving, downloading the first segment).\r\n\t/// </summary>\r\n\tTask<StreamSession> preparing;\r\n\tstring token;\r\n\tstring sessionClient;\r\n\tbool audioOnly;\r\n\tfloat startTime;\r\n\tbool disposed;\r\n\r\n\t/// <summary>\r\n\t/// Merged media: how long from starting the segmenter to its first segment being ready (download + write),\r\n\t/// averaged over every backend. A seek or join plans its first segment from where the synced time will be by then.\r\n\t/// </summary>\r\n\tstatic float firstSegmentSeconds = 1.5f;\r\n\tRealTimeSince sinceSegmenter;\r\n\tbool firstSegmentTimed;\r\n\r\n\t/// <summary> Checking / fixing an MP4 before playing it (see <see cref=\"Mp4ChannelFix\"/>). </summary>\r\n\tbool checkingMp4;\r\n\tstring localDirectory;\r\n\treadonly System.Threading.CancellationTokenSource cts = new();\r\n\r\n\t/// <summary>\r\n\t/// For streams that were started at an offset and whose clock starts at 0 - this is the offset.\r\n\t/// </summary>\r\n\tpublic float TimeOffset { get; private set; }\r\n\r\n\tpublic string Url { get; private set; }\r\n\tpublic bool IsAudioOnly => music is not null;\r\n\r\n\t/// <summary>\r\n\t/// Still resolving / downloading before anything can play.\r\n\t/// </summary>\r\n\tpublic bool IsPreparing => preparing is not null || checkingMp4;\r\n\r\n\t/// <summary>\r\n\t/// Something went wrong, user facing message.\r\n\t/// </summary>\r\n\tstring error;\r\n\tpublic string Error { get => error ?? live?.Error ?? segments?.Error; private set => error = value; }\r\n\r\n\t/// <summary> A segment build was refused (403/410) even after the url was refreshed - only a fresh session, from another client, helps. </summary>\r\n\tpublic bool Refused => segmenter?.Refused ?? false;\r\n\r\n\t/// <summary>\r\n\t/// Forget this media's cached session, so the next backend resolves fresh - skipping the client whose urls\r\n\t/// just refused (see MediaPlayer's refusal reload).\r\n\t/// </summary>\r\n\tpublic void InvalidateSession()\r\n\t{\r\n\t\tif ( token is not null ) StreamSessions.Forget( token, sessionClient );\r\n\t}\r\n\r\n\t/// <summary> Live: what we're connected to (\"RTSP H.264 1920x1080 + G.711 mu-law\"). </summary>\r\n\tpublic string LiveDescription => live is null ? null\r\n\t\t: $\"{live.Description} | behind newest frame {LiveBehindNewest:0.00}s{(LiveLatency is { } l ? $\", sent {l:0.00}s ago\" : \", no sender clock\")}{(live.Segmenter.Pcm is { } pcm ? $\", audio {pcm.SampleRate}Hz {pcm.Channels}ch, A/V {pcm.Offset * 1000:+0;-0}ms ({pcm.State})\" : \"\")}\";\r\n\r\n\t/// <summary> Live, decoded audio: its stream time minus the picture's (seconds, negative = audio late). </summary>\r\n\tpublic double? LiveAvOffset => live?.Segmenter.Pcm?.Offset;\r\n\r\n\t/// <summary>\r\n\t/// Live: how long ago the frame on screen was sent, by the sender's clock (RTCP sender reports) - the latency\r\n\t/// from the source's output to our screen. Null without sender reports (MPEG-TS, HLS, servers that send none).\r\n\t/// </summary>\r\n\tpublic double? LiveLatency => live?.Segmenter.WallClockZero is { } zero && segments is { Loaded: true }\r\n\t\t? (DateTime.UtcNow - DateTime.UnixEpoch).TotalSeconds - (zero + segments.StreamTime)\r\n\t\t: null;\r\n\r\n\t/// <summary> Live: stream time from the frame on screen to the newest frame received. </summary>\r\n\tpublic double LiveBehindNewest => live is null || segments is not { Loaded: true } ? 0 : live.Segmenter.NewestVideoTime - segments.StreamTime;\r\n\r\n\t/// <summary> Merged media opened before this media time starts from the beginning (see <see cref=\"StartNative\"/>). </summary>\r\n\tconst float FreshStartSeconds = 1.0f;\r\n\r\n\t/// <summary>\r\n\t/// Merged media: start at the keyframe before the planned time if it's at most this far behind it, else at the next\r\n\t/// one (waiting for it). Stays inside the drift tolerance (<see cref=\"MediaPlayer\"/>, 1.5 s) with the estimate's error.\r\n\t/// </summary>\r\n\tconst float MaxStartLag = 1.0f;\r\n\r\n\t/// <summary>\r\n\t/// Where the synced timeline is, while it's moving (null while paused). Merged media that starts at a later keyframe\r\n\t/// waits for it before opening its first segment.\r\n\t/// </summary>\r\n\tpublic float? SyncedTime { set { if ( segments is not null ) segments.SyncedTime = value; } }\r\n\r\n\t/// <summary>\r\n\t/// Playback has reached the end.\r\n\t/// </summary>\r\n\tbool finished;\r\n\tpublic bool Finished { get => finished || (segments?.Finished ?? false); private set => finished = value; }\r\n\r\n\tbool loaded;\r\n\tfloat lastPlaybackTime;\r\n\r\n\t/// <summary>\r\n\t/// Has the media actually started (metadata loaded / audio is advancing)?\r\n\t/// </summary>\r\n\tpublic bool Loaded => segments?.Loaded ?? (!IsPreparing && (loaded || PlaybackTime > 0.01f));\r\n\r\n\t/// <summary>\r\n\t/// Time since this backend was created.\r\n\t/// </summary>\r\n\tpublic RealTimeSince SinceCreated { get; private set; }\r\n\r\n\t/// <summary>\r\n\t/// Position within the media, taking the start offset into account.\r\n\t/// </summary>\r\n\tpublic float Time => TimeOffset + PlaybackTime;\r\n\r\n\tpublic float PlaybackTime => segments?.Time ?? video?.PlaybackTime ?? music?.PlaybackTime ?? 0;\r\n\r\n\t/// <summary>\r\n\t/// Duration as reported by the decoder.\r\n\t/// </summary>\r\n\tpublic float Duration => segments?.Duration ?? video?.Duration ?? music?.Duration ?? 0;\r\n\r\n\tpublic bool IsPaused => segments?.IsPaused ?? video?.IsPaused ?? music?.Paused ?? false;\r\n\r\n\tpublic Texture Texture => segments?.Texture ?? videoSink.Texture;\r\n\tpublic int Width => segments?.Width ?? (video is null ? 0 : videoSink.Width);\r\n\tpublic int Height => segments?.Height ?? (video is null ? 0 : videoSink.Height);\r\n\r\n\t/// <summary>\r\n\t/// Does the video have visible frames?\r\n\t/// </summary>\r\n\tpublic bool HasVideo => segments?.HasVideo ?? (video is not null && videoSink.Texture is not null && videoSink.Width > 4 && videoSink.Height > 4);\r\n\r\n\t/// <summary>\r\n\t/// 512 FFT values for visualizers. Empty for video playback.\r\n\t/// </summary>\r\n\tpublic ReadOnlySpan<float> Spectrum => music is not null ? music.Spectrum : ReadOnlySpan<float>.Empty;\r\n\r\n\tpublic float Amplitude => music?.Amplitude ?? 0;\r\n\r\n\t/// <summary>\r\n\t/// The decoded audio format, e.g. \"48000Hz 2ch\" (for diagnostics). Empty until audio has started.\r\n\t/// </summary>\r\n\tpublic string AudioFormat\r\n\t{\r\n\t\tget\r\n\t\t{\r\n\t\t\tvar (rate, ch) = segments is not null ? segments.AudioFormat : video is not null ? (video.SampleRate, video.Channels) : music is not null ? (music.SampleRate, music.Channels) : (0, 0);\r\n\t\t\treturn rate > 0 ? $\"{rate}Hz {ch}ch\" : \"\";\r\n\t\t}\r\n\t}\r\n\r\n\tMediaBackend() { }\r\n\r\n\t/// <summary>\r\n\t/// Start playing <paramref name=\"url\"/>. Never throws, check <see cref=\"Error\"/>.\r\n\t/// </summary>\r\n\tpublic static MediaBackend Create( string url, bool audioOnly, float timeOffset )\r\n\t{\r\n\t\tvar b = new MediaBackend\r\n\t\t{\r\n\t\t\tUrl = url,\r\n\t\t\tTimeOffset = timeOffset,\r\n\t\t\tSinceCreated = 0,\r\n\t\t};\r\n\r\n\t\tb.StartUrl( url, audioOnly );\r\n\t\treturn b;\r\n\t}\r\n\r\n\t/// <summary>\r\n\t/// Resolve a play token on this client and start playing it at <paramref name=\"startTime\"/> (media seconds).\r\n\t/// Never throws, check <see cref=\"Error\"/>.\r\n\t/// </summary>\r\n\tpublic static MediaBackend CreateNative( string token, bool audioOnly, float startTime )\r\n\t{\r\n\t\tvar b = new MediaBackend\r\n\t\t{\r\n\t\t\tUrl = token,\r\n\t\t\tSinceCreated = 0,\r\n\t\t\ttoken = token,\r\n\t\t\taudioOnly = audioOnly,\r\n\t\t\tstartTime = startTime,\r\n\t\t};\r\n\r\n\t\tb.preparing = StreamSessions.GetAsync( token );\r\n\t\treturn b;\r\n\t}\r\n\r\n\t/// <summary>\r\n\t/// Connect to a live stream (RTSP tunnelled over HTTP, MPEG-TS over HTTP, HLS) and play it near its live edge.\r\n\t/// Never throws, check <see cref=\"Error\"/>.\r\n\t/// </summary>\r\n\tpublic static MediaBackend CreateLive( string url, int maxHeight, bool audioOnly = false )\r\n\t{\r\n\t\tvar b = new MediaBackend { Url = url, SinceCreated = 0, audioOnly = audioOnly, liveMaxHeight = audioOnly ? AudioOnlyHeight : maxHeight };\r\n\t\ttry\r\n\t\t{\r\n\t\t\t// a live channel from an extractor (Twitch...): resolved on this client first (see StartNative)\r\n\t\t\tif ( PlayToken.IsToken( url ) )\r\n\t\t\t{\r\n\t\t\t\tb.token = url;\r\n\t\t\t\tb.preparing = StreamSessions.GetAsync( url );\r\n\t\t\t\treturn b;\r\n\t\t\t}\r\n\t\t\tb.StartLive( url, false );\r\n\t\t}\r\n\t\tcatch ( Exception e )\r\n\t\t{\r\n\t\t\tb.Error = e.Message;\r\n\t\t}\r\n\t\treturn b;\r\n\t}\r\n\r\n\tint liveMaxHeight = 720;\r\n\r\n\t/// <summary>\r\n\t/// The variant height an audio-only player (a speaker) asks a live or HLS stream for: the audio comes with every\r\n\t/// variant, so the smallest one saves the download and the decoding of a picture nobody sees.\r\n\t/// </summary>\r\n\tconst int AudioOnlyHeight = 144;\r\n\r\n\tvoid StartLive( string url, bool hls )\r\n\t{\r\n\t\tlive = new Resolver.Live.LiveStream( url, StreamSessions.NewCacheDirectory(), liveMaxHeight, hls );\r\n\t\tsegments = new SegmentPlayer( live.Segmenter );\r\n\t}\r\n\r\n\t/// <summary> The height a play token asks for (its pick, else the server's limit). </summary>\r\n\tstatic int TokenHeight( string token, int fallback )\r\n\t\t=> PlayToken.Parse( token ) is { } t ? (t.Height > 0 ? t.Height : t.MaxHeight) : fallback;\r\n\r\n\tvoid StartUrl( string url, bool audio )\r\n\t{\r\n\t\t// MP4s whose header mislabels mono audio as stereo crackle in the engine - check first\r\n\t\tif ( Mp4ChannelFix.Applies( url ) )\r\n\t\t{\r\n\t\t\t_ = StartMp4( url, audio );\r\n\t\t\treturn;\r\n\t\t}\r\n\r\n\t\tPlayUrlNow( url, audio );\r\n\t}\r\n\r\n\tasync Task StartMp4( string url, bool audio )\r\n\t{\r\n\t\tcheckingMp4 = true;\r\n\t\ttry\r\n\t\t{\r\n\t\t\tlocalDirectory = StreamSessions.NewCacheDirectory();\r\n\t\t\tvar path = await Mp4ChannelFix.PrepareAsync( url, localDirectory, cts.Token );\r\n\t\t\tif ( disposed ) return;\r\n\r\n\t\t\tif ( path is null ) PlayUrlNow( url, audio );\r\n\t\t\telse PlayFile( path, audio );\r\n\t\t}\r\n\t\tcatch ( OperationCanceledException )\r\n\t\t{\r\n\t\t}\r\n\t\tcatch ( Exception e )\r\n\t\t{\r\n\t\t\tif ( disposed ) return;\r\n\t\t\t// couldn't fix it - play it as it is rather than not at all\r\n\t\t\tLog.Warning( $\"[bimp] Couldn't prepare {url}: {e.Message}\" );\r\n\t\t\tPlayUrlNow( url, audio );\r\n\t\t}\r\n\t\tfinally\r\n\t\t{\r\n\t\t\tcheckingMp4 = false;\r\n\t\t}\r\n\t}\r\n\r\n\tvoid PlayFile( string path, bool audio )\r\n\t{\r\n\t\ttry\r\n\t\t{\r\n\t\t\tif ( audio )\r\n\t\t\t{\r\n\t\t\t\tmusic = MusicPlayer.Play( FileSystem.Data, path );\r\n\t\t\t\tmusic.OnFinished += () => Finished = true;\r\n\t\t\t}\r\n\t\t\telse\r\n\t\t\t{\r\n\t\t\t\tvideo = new VideoPlayer();\r\n\t\t\t\tvideoSink.Attach( video, null, null, \"video\" );\r\n\t\t\t\tvideo.OnLoaded += () => loaded = true;\r\n\t\t\t\tvideo.OnFinished += () => Finished = true;\r\n\t\t\t\tvideo.Play( FileSystem.Data, path );\r\n\t\t\t}\r\n\t\t}\r\n\t\tcatch ( Exception e )\r\n\t\t{\r\n\t\t\tError = e.Message;\r\n\t\t\tLog.Warning( $\"[bimp] Failed to play {path}: {e.Message}\" );\r\n\t\t}\r\n\t}\r\n\r\n\tvoid PlayUrlNow( string url, bool audio )\r\n\t{\r\n\t\ttry\r\n\t\t{\r\n\t\t\tif ( audio )\r\n\t\t\t{\r\n\t\t\t\tmusic = MusicPlayer.PlayUrl( url );\r\n\t\t\t\tmusic.OnFinished += () => Finished = true;\r\n\t\t\t}\r\n\t\t\telse\r\n\t\t\t{\r\n\t\t\t\tvideo = new VideoPlayer();\r\n\t\t\t\tvideoSink.Attach( video, null, null, \"video\" );\r\n\t\t\t\tvideo.OnLoaded += () => loaded = true;\r\n\t\t\t\tvideo.OnFinished += () => Finished = true;\r\n\t\t\t\tvideo.Play( url );\r\n\t\t\t}\r\n\t\t\t_ = CheckReachable( url );\r\n\t\t}\r\n\t\tcatch ( Exception e )\r\n\t\t{\r\n\t\t\tError = e.Message;\r\n\t\t\tLog.Warning( $\"[bimp] Failed to play {url}: {e.Message}\" );\r\n\t\t}\r\n\t}\r\n\r\n\t/// <summary>\r\n\t/// The engine's players give up on a url that answers with an error page (a 403, a 404) without saying so - the\r\n\t/// screen would sit on \"loading\" for good. Look at the response's status ourselves. Headers only: a radio stream\r\n\t/// never ends, and a buffered request would never return.\r\n\t/// </summary>\r\n\tasync Task CheckReachable( string url )\r\n\t{\r\n\t\tif ( !url.StartsWith( \"http\", StringComparison.OrdinalIgnoreCase ) ) return;\r\n\t\ttry\r\n\t\t{\r\n\t\t\tusing var stream = await Http.RequestStreamAsync( url, cancellationToken: cts.Token );\r\n\t\t}\r\n\t\tcatch ( System.Net.Http.HttpRequestException e ) when ( e.StatusCode is { } status && !loaded && !disposed )\r\n\t\t{\r\n\t\t\tError = $\"The server refused the file ({(int)status}).\";\r\n\t\t\tLog.Warning( $\"[bimp] {url}: {(int)status}\" );\r\n\t\t}\r\n\t\tcatch ( Exception )\r\n\t\t{\r\n\t\t\t// unreachable, cancelled: the engine's own load decides\r\n\t\t}\r\n\t}\r\n\r\n\t/// <summary>\r\n\t/// The client side resolve finished - start the right kind of playback.\r\n\t/// </summary>\r\n\tvoid StartNative( StreamSession session )\r\n\t{\r\n\t\tvar plan = session.Plan;\r\n\t\tsessionClient = plan.ClientKey;\r\n\r\n\t\tif ( plan.Kind == StreamKind.Direct )\r\n\t\t{\r\n\t\t\t// a progressive file - played straight from the site, seeked natively once loaded\r\n\t\t\tStartUrl( plan.DirectUrl, audioOnly );\r\n\t\t\treturn;\r\n\t\t}\r\n\r\n\t\tif ( plan.Kind == StreamKind.Live )\r\n\t\t{\r\n\t\t\tliveMaxHeight = audioOnly ? AudioOnlyHeight : TokenHeight( token, liveMaxHeight );\r\n\t\t\tStartLive( plan.DirectUrl, true );\r\n\t\t\treturn;\r\n\t\t}\r\n\r\n\t\t// The engine can only seek forward by decoding every frame in between, while the audio runs on - a 4K video\r\n\t\t// can't decode much faster than it plays, so after a seek the picture crawled behind the sound for seconds. So\r\n\t\t// merged media starts at a keyframe close behind where the synced time will be once the first segment is\r\n\t\t// ready, or at the next one and waits there (see SegmentPlayer.SyncedTime) - it never seeks forward. The start\r\n\t\t// of a video just plays from the beginning, a load time behind.\r\n\t\tvar planAhead = plan.Kind is StreamKind.Merge or StreamKind.Hls && !audioOnly && startTime >= FreshStartSeconds;\r\n\t\tvar plannedStart = planAhead ? startTime + (float)SinceCreated + firstSegmentSeconds : startTime;\r\n\t\tvar maxLag = planAhead ? MaxStartLag : float.MaxValue;\r\n\t\tsinceSegmenter = 0;\r\n\r\n\t\tif ( plan.Kind == StreamKind.Hls )\r\n\t\t{\r\n\t\t\t// a finished HLS playlist: remuxed from the segment at (or just after) the planned start, like merged media\r\n\t\t\tlive = new Resolver.Live.LiveStream( plan.DirectUrl, StreamSessions.NewCacheDirectory(), audioOnly ? AudioOnlyHeight : TokenHeight( token, 720 ), true, plannedStart, maxLag );\r\n\t\t\tsegments = new SegmentPlayer( live.Segmenter );\r\n\t\t\treturn;\r\n\t\t}\r\n\r\n\t\tsegmenter = new WebmSegmenter( plan.Kind == StreamKind.Merge ? session.Video : null, session.Audio, plannedStart, StreamSessions.NewCacheDirectory(), maxLag, session.SwitchVideo );\r\n\r\n\t\tif ( plan.Kind == StreamKind.Audio || audioOnly )\r\n\t\t{\r\n\t\t\t// one file from the cue point before the start time to the end, through MusicPlayer for the spectrum\r\n\t\t\t_ = StartAudioFile();\r\n\t\t\treturn;\r\n\t\t}\r\n\r\n\t\tsegments = new SegmentPlayer( segmenter );\r\n\t}\r\n\r\n\tasync Task StartAudioFile()\r\n\t{\r\n\t\tpreparing = null;\r\n\t\ttry\r\n\t\t{\r\n\t\t\tvar segment = await segmenter.GetAsync( 0 );\r\n\t\t\tif ( disposed || segment is null ) return;\r\n\t\t\tmusic = MusicPlayer.Play( FileSystem.Data, segment.Path );\r\n\t\t\tmusic.OnFinished += () => Finished = true;\r\n\t\t}\r\n\t\tcatch ( Exception e )\r\n\t\t{\r\n\t\t\tif ( disposed ) return;\r\n\t\t\tError = e.Message;\r\n\t\t\tStreamSessions.Forget( token );\r\n\t\t}\r\n\t}\r\n\r\n\t/// <summary>\r\n\t/// Call every frame. Pushes the latest video frame to the texture.\r\n\t/// </summary>\r\n\tpublic void Present()\r\n\t{\r\n\t\tif ( preparing is { IsCompleted: true } task )\r\n\t\t{\r\n\t\t\tpreparing = null;\r\n\t\t\tif ( task.IsCompletedSuccessfully )\r\n\t\t\t{\r\n\t\t\t\ttry\r\n\t\t\t\t{\r\n\t\t\t\t\tStartNative( task.Result );\r\n\t\t\t\t}\r\n\t\t\t\tcatch ( Exception e )\r\n\t\t\t\t{\r\n\t\t\t\t\tError = e.Message;\r\n\t\t\t\t\tStreamSessions.Forget( token );\r\n\t\t\t\t}\r\n\t\t\t}\r\n\t\t\telse\r\n\t\t\t{\r\n\t\t\t\tvar e = task.Exception?.InnerException;\r\n\t\t\t\tError = e is ResolveException ? e.Message : $\"Couldn't load the stream: {e?.Message}\";\r\n\t\t\t\tLog.Warning( $\"[bimp] Failed to resolve {token}: {e}\" );\r\n\t\t\t\tStreamSessions.Forget( token );\r\n\t\t\t}\r\n\t\t}\r\n\r\n\t\tif ( !firstSegmentTimed && ((ISegmentFeed)segmenter ?? (live?.Segmenter is { IsLive: false } vod ? vod : null))?.TryGet( 0 ) is not null )\r\n\t\t{\r\n\t\t\tfirstSegmentTimed = true;\r\n\t\t\tfirstSegmentSeconds = Math.Clamp( firstSegmentSeconds * 0.5f + sinceSegmenter * 0.5f, 0.3f, 10.0f );\r\n\t\t}\r\n\r\n\t\tsegments?.Update();\r\n\t\tif ( video is not null )\r\n\t\t{\r\n\t\t\tvar presentStart = System.Diagnostics.Stopwatch.GetTimestamp();\r\n\t\t\tvideo.Present();\r\n\t\t\tMediaProbe.PresentTook( System.Diagnostics.Stopwatch.GetElapsedTime( presentStart ).TotalMilliseconds, \"direct\", 0 );\r\n\t\t}\r\n\r\n\t\tif ( live is not null && segments is not null )\r\n\t\t{\r\n\t\t\tlive.PlayingSeq = segments.CurrentSeq;\r\n\t\t\t// camera G.711 audio plays beside the video, lined up with the picture on screen\r\n\t\t\tlive.Segmenter.Pcm?.Update( segments.StreamTime, segments.Loaded && requestedPaused != true );\r\n\t\t}\r\n\r\n\t\t// MusicPlayer has no loaded event, so treat advancing time as loaded\r\n\t\tvar t = PlaybackTime;\r\n\t\tif ( t > lastPlaybackTime + 0.01f ) loaded = true;\r\n\t\tlastPlaybackTime = t;\r\n\t}\r\n\r\n\tbool? requestedPaused;\r\n\tRealTimeSince sincePauseRequest;\r\n\r\n\t/// <summary>\r\n\t/// Pause or resume. Only calls into the native player when the requested state changes - not by\r\n\t/// polling its IsPaused every frame, which can lag a call and made us call Pause/Resume repeatedly\r\n\t/// (every extra Resume restarts the decoder's clock, which shows up as stutter).\r\n\t/// If the native player still disagrees a while later (e.g. it hadn't started yet), ask once more.\r\n\t/// </summary>\r\n\tpublic void SetPaused( bool paused )\r\n\t{\r\n\t\tif ( requestedPaused == paused )\r\n\t\t{\r\n\t\t\tif ( IsPaused == paused || sincePauseRequest < 1.0f ) return;\r\n\t\t}\r\n\r\n\t\trequestedPaused = paused;\r\n\t\tsincePauseRequest = 0;\r\n\t\tMediaProbe.Note( paused ? \"native Pause()\" : \"native Resume()\" );\r\n\r\n\t\tsegments?.SetPaused( paused );\r\n\r\n\t\tif ( video is not null )\r\n\t\t{\r\n\t\t\tif ( paused ) video.Pause(); else video.Resume();\r\n\t\t}\r\n\r\n\t\tif ( music is not null )\r\n\t\t\tmusic.Paused = paused;\r\n\t}\r\n\r\n\t/// <summary>\r\n\t/// Seek to a position within the media (not accounting for <see cref=\"TimeOffset\"/> - pass media time).\r\n\t/// </summary>\r\n\tpublic void Seek( float mediaTime )\r\n\t{\r\n\t\tvar local = MathF.Max( 0, mediaTime - TimeOffset );\r\n\t\tMediaProbe.Note( $\"native Seek({local:0.000}) from {PlaybackTime:0.000}\" );\r\n\t\tsegments?.Seek( local );\r\n\t\tvideo?.Seek( local );\r\n\t\tmusic?.Seek( local );\r\n\t\tFinished = false;\r\n\t}\r\n\r\n\tpublic void SetAudio( Vector3 position, bool spatial, float volume, float distance )\r\n\t{\r\n\t\tsegments?.SetAudio( position, spatial, volume, distance );\r\n\t\tlive?.Segmenter.Pcm?.SetAudio( position, spatial, volume, distance );\r\n\r\n\t\t// Non-spatial audio is played at the listener, slightly in front so it's centered\r\n\t\tvar pos = spatial ? position : Vector3.Forward * 64.0f;\r\n\r\n\t\tif ( video is not null )\r\n\t\t{\r\n\t\t\tvar a = video.Audio;\r\n\t\t\ta.ListenLocal = !spatial;\r\n\t\t\ta.Position = pos;\r\n\t\t\ta.Volume = volume;\r\n\t\t\ta.Distance = distance;\r\n\t\t\ta.Falloff = MediaPlayer.AudioFalloff;\r\n\t\t}\r\n\r\n\t\tif ( music is not null )\r\n\t\t{\r\n\t\t\tmusic.ListenLocal = !spatial;\r\n\t\t\tmusic.Position = pos;\r\n\t\t\tmusic.Volume = volume;\r\n\t\t\tmusic.Distance = distance;\r\n\t\t\tmusic.Falloff = MediaPlayer.AudioFalloff;\r\n\t\t}\r\n\t}\r\n\r\n\tpublic void Dispose()\r\n\t{\r\n\t\tdisposed = true;\r\n\t\tpreparing = null;\r\n\t\tcts.Cancel();\r\n\r\n\t\tsegments?.Dispose();\r\n\t\tsegments = null;\r\n\r\n\t\tvideo?.Dispose();\r\n\t\tvideo = null;\r\n\r\n\t\tmusic?.Dispose();\r\n\t\tmusic = null;\r\n\r\n\t\tvideoSink.Dispose();\r\n\r\n\t\t// after the players, so the files aren't open anymore\r\n\t\tsegmenter?.Dispose();\r\n\t\tsegmenter = null;\r\n\r\n\t\tlive?.Dispose();\r\n\t\tlive = null;\r\n\r\n\t\tif ( localDirectory is not null )\r\n\t\t{\r\n\t\t\ttry\r\n\t\t\t{\r\n\t\t\t\tif ( FileSystem.Data.DirectoryExists( localDirectory ) ) FileSystem.Data.DeleteDirectory( localDirectory, true );\r\n\t\t\t}\r\n\t\t\tcatch ( Exception e )\r\n\t\t\t{\r\n\t\t\t\tLog.Trace( $\"[bimp] couldn't delete {localDirectory}: {e.Message}\" );\r\n\t\t\t}\r\n\t\t\tlocalDirectory = null;\r\n\t\t}\r\n\t}\r\n}\r\n"
},
{
"Ident": "rexi.bimp",
"Path": "MediaInteract.cs",
"FileName": "MediaInteract.cs",
"PackageType": "library",
"CodeKind": "Game",
"AssetVersionId": 399292,
"IsPrivate": false,
"Code": "namespace Bimp;\n\n/// <summary>\n/// Lets players use the media player with the USE key (works with the engine's <see cref=\"PlayerController\"/>,\n/// or anything else that calls <see cref=\"Component.IPressable\"/>). Needs a collider on this GameObject or a child.\n/// <para>\n/// Aiming at one of the screen's on-screen buttons while the controls are showing presses that button.\n/// Anywhere else (or while the controls are hidden) opens the full remote.\n/// </para>\n/// </summary>\n[Title( \"Media Player Interact\" ), Category( \"Media\" ), Icon( \"touch_app\" )]\npublic sealed class MediaInteract : Component, Component.IPressable\n{\n\t/// <summary>\n\t/// The player to control. Found automatically in this object or its parents if unset.\n\t/// </summary>\n\t[Property] public MediaPlayer Player { get; set; }\n\n\t/// <summary>\n\t/// The screen whose on-screen buttons USE can press. Found automatically in this object or its children if unset.\n\t/// </summary>\n\t[Property] public MediaScreen Screen { get; set; }\n\n\tMediaPlayer Target => Player.IsValid() ? Player : (Player = GetComponentInParent<MediaPlayer>());\n\tMediaScreen TargetScreen => Screen.IsValid() ? Screen : (Screen = GetComponentInChildren<MediaScreen>());\n\n\tpublic bool CanPress( IPressable.Event e ) => Target.IsValid();\n\n\tpublic bool Press( IPressable.Event e )\n\t{\n\t\tif ( !Target.IsValid() ) return false;\n\n\t\t// The screen may already have handled this USE press itself this frame\n\t\tif ( !UseHandled.TryConsume() ) return true;\n\n\t\t// Press runs on the client that pressed, so this only affects them (button presses go through the\n\t\t// player's permission-checked host requests)\n\t\tvar screen = TargetScreen;\n\t\tif ( screen.IsValid() && screen.TryPressAimedControl() )\n\t\t\treturn true;\n\n\t\tMediaRemote.Toggle( Target );\n\t\treturn true;\n\t}\n\n\tpublic IPressable.Tooltip? GetTooltip( IPressable.Event e )\n\t{\n\t\tif ( !Target.IsValid() ) return null;\n\n\t\tvar title = Target.HasMedia ? Target.Title : \"Media Player\";\n\t\tvar icon = Target.AudioOnlyPlayer ? \"speaker\" : \"smart_display\";\n\n\t\tvar action = TargetScreen.IsValid() ? TargetScreen.AimedActionName : null;\n\t\tif ( action is not null )\n\t\t\treturn new IPressable.Tooltip( action, \"touch_app\", title );\n\n\t\treturn new IPressable.Tooltip( title, icon, \"Open controls\" );\n\t}\n}\n"
},
{
"Ident": "rexi.bimp",
"Path": "Resolver/FormatSelector.cs",
"FileName": "FormatSelector.cs",
"PackageType": "library",
"CodeKind": "Game",
"AssetVersionId": 399292,
"IsPrivate": false,
"Code": "namespace Bimp.Resolver;\n\n/// <summary>\n/// One format an extractor found.\n/// </summary>\npublic sealed class MediaFormat\n{\n\tpublic string Id { get; set; }\n\tpublic string Url { get; set; }\n\n\t/// <summary> \"webm\", \"mp4\", \"mp3\"... </summary>\n\tpublic string Container { get; set; }\n\n\t/// <summary> Lowercase codec name, null if the format has no video. </summary>\n\tpublic string VideoCodec { get; set; }\n\n\t/// <summary> Lowercase codec name, null if the format has no audio. </summary>\n\tpublic string AudioCodec { get; set; }\n\n\tpublic int Height { get; set; }\n\tpublic float Fps { get; set; }\n\tpublic long Bitrate { get; set; }\n\tpublic long Size { get; set; }\n\tpublic bool Hdr { get; set; }\n\n\t/// <summary> A dynamic range compressed copy of an audio track. </summary>\n\tpublic bool Drc { get; set; }\n\n\t/// <summary> Audio language (\"en\", \"es-419\"), null if unlabelled. </summary>\n\tpublic string Language { get; set; }\n\tpublic string LanguageName { get; set; }\n\n\t/// <summary> This is the original audio track. </summary>\n\tpublic bool LanguageDefault { get; set; }\n\n\tpublic bool HasVideo => VideoCodec is not null;\n\tpublic bool HasAudio => AudioCodec is not null;\n\tpublic bool IsVp9OrAv1 => VideoCodec is { } v && (v.StartsWith( \"vp9\" ) || v.StartsWith( \"vp09\" ) || v.StartsWith( \"av01\" ) || v.StartsWith( \"av1\" ));\n\tpublic bool IsOpus => AudioCodec?.StartsWith( \"opus\" ) ?? false;\n\n\tpublic StreamFile ToFile() => new() { Url = Url, Size = Size, FormatId = Id, Container = Container };\n}\n\n/// <summary>\n/// Picks formats the engine can play. A port of the old resolver's format picking:\n/// VP9/AV1 WebM video + Opus WebM audio (merged locally), else a single muxed file.\n/// </summary>\npublic static class FormatSelector\n{\n\t/// <summary> Video-only VP9/AV1 WebM files - the ones we can merge without re-encoding. </summary>\n\tpublic static List<MediaFormat> WebmVideo( IEnumerable<MediaFormat> formats )\n\t\t=> formats.Where( f => f.Container == \"webm\" && f.IsVp9OrAv1 && !f.HasAudio ).ToList();\n\n\t/// <summary> Audio-only Opus WebM files (every language), without the dynamic-range-compressed copies. </summary>\n\tpublic static List<MediaFormat> WebmAudio( IEnumerable<MediaFormat> formats )\n\t\t=> formats.Where( f => f.Container == \"webm\" && f.IsOpus && !f.HasVideo && !f.Drc ).ToList();\n\n\t/// <summary>\n\t/// Tallest at or below the limit first, SDR before HDR (the engine's HDR output is untested and would look\n\t/// washed out anyway), then fps, then bitrate.\n\t/// </summary>\n\tstatic (bool, bool, int, float, long) ScoreVideo( MediaFormat f, int limit )\n\t\t=> (f.Height <= limit, !f.Hdr, f.Height <= limit ? f.Height : -f.Height, f.Fps, f.Bitrate);\n\n\t/// <summary>\n\t/// VP9 at 4K and more than 30 fps: more than the engine's VP9 decoder keeps up with (measured 19-45 frames a second\n\t/// on a Ryzen 9 9900X3D; 1440p60 and 4K30 are fine).\n\t/// </summary>\n\tpublic static bool TooHeavyForVp9( MediaFormat f ) => f.Height >= 2160 && f.Fps > 30;\n\n\t/// <summary> An 8-bit SDR AV1 video-only MP4 with the same height and frame rate, highest bitrate first. </summary>\n\tpublic static MediaFormat Av1Like( IEnumerable<MediaFormat> formats, MediaFormat like )\n\t\t=> formats.Where( f => f.Container == \"mp4\" && !f.HasAudio && !f.Hdr && f.Height == like.Height && MathF.Abs( f.Fps - like.Fps ) < 1\n\t\t\t\t&& f.VideoCodec is { } v && v.StartsWith( \"av01\" ) && v.Split( '.' ) is { Length: >= 4 } p && p[3] == \"08\" )\n\t\t\t.OrderByDescending( f => f.Bitrate )\n\t\t\t.FirstOrDefault();\n\n\t/// <summary>\n\t/// What a 4K60 AV1 choice carries on with once YouTube stops serving it: the tallest SDR VP9 below 4K at the same\n\t/// frame rate (1440p60 - the engine's VP9 decoder keeps up with that).\n\t/// </summary>\n\tpublic static MediaFormat SwitchLike( IEnumerable<MediaFormat> formats, MediaFormat like )\n\t\t=> WebmVideo( formats ).Where( f => !f.Hdr && f.Height < 2160 && MathF.Abs( f.Fps - like.Fps ) < 1 && f.VideoCodec is { } v && !v.StartsWith( \"av01\" ) )\n\t\t\t.OrderByDescending( f => f.Height ).ThenByDescending( f => f.Bitrate )\n\t\t\t.FirstOrDefault();\n\n\tpublic static List<int> Qualities( IEnumerable<MediaFormat> formats )\n\t{\n\t\tvar video = WebmVideo( formats );\n\t\tvar sdr = video.Where( f => !f.Hdr ).ToList();\n\t\tif ( sdr.Count == 0 ) sdr = video;\n\t\treturn sdr.Select( f => f.Height ).Where( h => h > 0 ).Distinct().OrderByDescending( h => h ).ToList();\n\t}\n\n\t/// <summary> The original audio track's language. </summary>\n\tstatic string DefaultLanguage( List<MediaFormat> audio )\n\t{\n\t\tvar labelled = audio.Where( f => !string.IsNullOrEmpty( f.Language ) ).ToList();\n\t\tif ( labelled.Count == 0 ) return null;\n\t\treturn (labelled.FirstOrDefault( f => f.LanguageDefault ) ?? labelled[0]).Language;\n\t}\n\n\tpublic static List<MediaAudioTrack> AudioTracks( IEnumerable<MediaFormat> formats )\n\t{\n\t\tvar audio = WebmAudio( formats );\n\t\tvar def = DefaultLanguage( audio );\n\t\tvar tracks = new Dictionary<string, MediaAudioTrack>();\n\t\tforeach ( var f in audio )\n\t\t{\n\t\t\tif ( string.IsNullOrEmpty( f.Language ) || tracks.ContainsKey( f.Language ) ) continue;\n\t\t\ttracks[f.Language] = new MediaAudioTrack { Id = f.Language, Name = f.LanguageName ?? f.Language, IsDefault = f.Language == def };\n\t\t}\n\n\t\t// a single (or unlabelled) track isn't a choice\n\t\tif ( tracks.Count < 2 ) return new List<MediaAudioTrack>();\n\t\treturn tracks.Values.OrderBy( t => !t.IsDefault ).ThenBy( t => t.Name.ToLowerInvariant() ).ToList();\n\t}\n\n\t/// <summary> Do these formats have an audio track in this language, matched the way <see cref=\"PickAudio\"/> matches it? </summary>\n\tpublic static bool HasLanguage( IEnumerable<MediaFormat> formats, string lang )\n\t{\n\t\tif ( string.IsNullOrWhiteSpace( lang ) ) return true;\n\t\tvar primary = lang.Split( '-' )[0];\n\t\treturn WebmAudio( formats ).Any( f => string.Equals( f.Language, lang, StringComparison.OrdinalIgnoreCase )\n\t\t\t|| string.Equals( (f.Language ?? \"\").Split( '-' )[0], primary, StringComparison.OrdinalIgnoreCase ) );\n\t}\n\n\t/// <summary> Audio formats for the requested language (\"es\" finds \"es-419\"), falling back to the original track. </summary>\n\tstatic List<MediaFormat> PickAudio( List<MediaFormat> audio, string lang )\n\t{\n\t\tif ( audio.Count == 0 ) return audio;\n\n\t\tif ( !string.IsNullOrWhiteSpace( lang ) )\n\t\t{\n\t\t\tvar exact = audio.Where( f => string.Equals( f.Language, lang, StringComparison.OrdinalIgnoreCase ) ).ToList();\n\t\t\tif ( exact.Count > 0 ) return exact;\n\n\t\t\tvar primary = lang.Split( '-' )[0];\n\t\t\tvar prefix = audio.Where( f => string.Equals( (f.Language ?? \"\").Split( '-' )[0], primary, StringComparison.OrdinalIgnoreCase ) ).ToList();\n\t\t\tif ( prefix.Count > 0 ) return prefix;\n\t\t}\n\n\t\tvar def = DefaultLanguage( audio );\n\t\treturn def is null ? audio : audio.Where( f => f.Language == def ).ToList();\n\t}\n\n\t/// <summary>\n\t/// Best first. With an exact height, that height wins if it exists, else the nearest below it.\n\t/// </summary>\n\tstatic List<MediaFormat> PickVideo( List<MediaFormat> video, int maxHeight, int height )\n\t{\n\t\tvar limit = height > 0 ? height : maxHeight;\n\t\treturn video.OrderByDescending( f => ScoreVideo( f, limit ) ).ToList();\n\t}\n\n\tpublic static StreamPlan Plan( IReadOnlyList<MediaFormat> formats, StreamRequest request )\n\t{\n\t\tvar maxh = Math.Clamp( request.MaxHeight, 144, 2160 );\n\t\tvar audio = PickAudio( WebmAudio( formats ), request.Language ).OrderByDescending( f => f.Bitrate ).ToList();\n\n\t\tif ( request.AudioOnly )\n\t\t{\n\t\t\tif ( audio.Count > 0 )\n\t\t\t\treturn new StreamPlan { Kind = StreamKind.Audio, Audio = audio[0].ToFile() };\n\n\t\t\t// anything with audio the engine can decode on its own\n\t\t\tvar anyAudio = formats.Where( f => f.HasAudio && !f.HasVideo && f.Container is \"mp3\" or \"ogg\" or \"m4a\" )\n\t\t\t\t.OrderByDescending( f => f.Bitrate ).FirstOrDefault()\n\t\t\t\t?? formats.Where( f => f.HasAudio && f.Container is \"mp4\" or \"webm\" ).OrderBy( f => f.Height ).FirstOrDefault();\n\t\t\tif ( anyAudio is not null )\n\t\t\t\treturn new StreamPlan { Kind = StreamKind.Direct, DirectUrl = anyAudio.Url };\n\n\t\t\tthrow new ResolveException( \"No playable audio found.\" );\n\t\t}\n\n\t\tvar video = PickVideo( WebmVideo( formats ), maxh, request.Height );\n\t\tif ( video.Count > 0 && audio.Count > 0 )\n\t\t\treturn new StreamPlan { Kind = StreamKind.Merge, Video = video[0].ToFile(), Audio = audio[0].ToFile() };\n\n\t\t// a single muxed file the engine can decode (VP9/AV1+Opus WebM, or H.264/AAC MP4)\n\t\tvar muxed = formats.Where( f => f.HasVideo && f.HasAudio && f.Container is \"webm\" or \"mp4\" )\n\t\t\t.OrderByDescending( f => ScoreVideo( f, maxh ) )\n\t\t\t.ThenByDescending( f => f.Container == \"webm\" )\n\t\t\t.FirstOrDefault();\n\t\tif ( muxed is not null )\n\t\t\treturn new StreamPlan { Kind = StreamKind.Direct, DirectUrl = muxed.Url };\n\n\t\tthrow new ResolveException( \"No playable formats found.\" );\n\t}\n}\n"
}
]
}