diff --git a/ErsatzTV.FFmpeg.Tests/PipelineBuilderBaseTests.cs b/ErsatzTV.FFmpeg.Tests/PipelineBuilderBaseTests.cs index 64e485b4e..53f4114e9 100644 --- a/ErsatzTV.FFmpeg.Tests/PipelineBuilderBaseTests.cs +++ b/ErsatzTV.FFmpeg.Tests/PipelineBuilderBaseTests.cs @@ -1,5 +1,6 @@ -using System; +using System; using System.Collections.Generic; +using System.Text.RegularExpressions; using ErsatzTV.FFmpeg.Capabilities; using ErsatzTV.FFmpeg.Encoder; using ErsatzTV.FFmpeg.Format; @@ -531,6 +532,168 @@ public class PipelineBuilderBaseTests "-nostdin -hide_banner -nostats -loglevel error -i /test/input/file.png -vf scale=-1:200:force_original_aspect_ratio=decrease /test/output/file.jpg"); } + [Test] + public void Realtime_Input_Should_Burst_Initial_Segments_When_Option_Is_Supported() + { + string command = BuildRealtimeCommand(new BurstCapableFFmpegCapabilities()); + + // burst covers the first two 4s segments, then the 1.05 throttle resumes (ersatztv#350). + // anchor on the input path so this can't be satisfied by some other input carrying the + // burst; audio and video share one file here, so exactly one input is expected + command.ShouldContain("-readrate 1.05 -readrate_initial_burst 8 -i /tmp/whatever.mkv"); + Regex.Matches(command, "-readrate_initial_burst 8").Count.ShouldBe(1); + } + + [Test] + public void Realtime_Input_Should_Not_Burst_When_Option_Is_Unsupported() + { + string command = BuildRealtimeCommand(new DefaultFFmpegCapabilities()); + + command.ShouldContain("-readrate 1.05 -i"); + command.ShouldNotContain("-readrate_initial_burst"); + } + + [Test] + public void Realtime_Input_Should_Not_Burst_A_Still_Image() + { + // a still image is paced by the realtime filter, so bursting would only run audio ahead + string command = BuildRealtimeCommand(new BurstCapableFFmpegCapabilities(), stillImage: true); + + // the positive anchor keeps this from passing vacuously if the helper ever stops + // producing a realtime audio input at all + command.ShouldContain("-readrate 1.05"); + command.ShouldNotContain("-readrate_initial_burst"); + } + + [Test] + public void Concat_Should_Never_Burst() + { + // concat reads already-written segments from the running segmenter; bursting would + // gallop through them + var concatInputFile = new ConcatInputFile("http://localhost:8080/ffmpeg/concat/1", new FrameSize(1920, 1080)); + + var builder = new SoftwarePipelineBuilder( + new BurstCapableFFmpegCapabilities(), + HardwareAccelerationMode.None, + None, + None, + None, + None, + concatInputFile, + Option.None, + "", + "", + _logger); + + FFmpegPipeline result = builder.Concat(concatInputFile, FFmpegState.Concat(false, "Some Channel")); + + string command = PrintCommand(None, None, None, concatInputFile, None, result); + + command.ShouldContain("-readrate 1.0"); + command.ShouldNotContain("-readrate_initial_burst"); + } + + private string BuildRealtimeCommand(IFFmpegCapabilities capabilities, bool stillImage = false) + { + var videoInputFile = new VideoInputFile( + "/tmp/whatever.mkv", + new List + { + new( + 0, + VideoFormat.H264, + VideoProfile.Main, + new PixelFormatYuv420P(), + ColorParams.Default, + new FrameSize(1920, 1080), + "1:1", + "16:9", + FrameRate.DefaultFrameRate, + stillImage, + ScanKind.Progressive) + }); + + var desiredState = new FrameState( + true, + false, + VideoFormat.Hevc, + VideoProfile.Main, + VideoPreset.Unset, + false, + new PixelFormatYuv420P(), + new FrameSize(1920, 1080), + new FrameSize(1920, 1080), + Option.None, + FFmpegFilterMode.HardwareIfPossible, + false, + Option.None, + 2000, + 4000, + 90_000, + false, + false); + + var ffmpegState = new FFmpegState( + false, + HardwareAccelerationMode.None, + HardwareAccelerationMode.None, + Option.None, + Option.None, + TimeSpan.FromSeconds(1), + Option.None, + false, + Option.None, + Option.None, + Option.None, + Option.None, + Option.None, + OutputFormatKind.MpegTs, + Option.None, + Option.None, + Option.None, + Option.None, + TimeSpan.Zero, + Option.None, + Option.None, + false, + false, + "clip", + false); + + // a *separate* audio input matters here: for a still image the video input takes no readrate + // at all, so only a distinct audio input can prove the burst was suppressed (this is the + // song shape — cover art plus an audio file) + var audioInputFile = new AudioInputFile( + stillImage ? "/tmp/whatever.mp3" : "/tmp/whatever.mkv", + new List { new(1, AudioFormat.Aac, 2) }, + new AudioState( + AudioFormat.Aac, + 2, + 320, + 640, + 48, + false, + AudioFilter.None, + Option.None)); + + var builder = new SoftwarePipelineBuilder( + capabilities, + HardwareAccelerationMode.None, + videoInputFile, + audioInputFile, + None, + None, + None, + Option.None, + "", + "", + _logger); + + FFmpegPipeline result = builder.Build(ffmpegState, desiredState); + + return PrintCommand(videoInputFile, audioInputFile, None, None, None, result); + } + private static string PrintCommand( Option videoInputFile, Option audioInputFile, @@ -563,4 +726,13 @@ public class PipelineBuilderBaseTests new System.Collections.Generic.HashSet(), new System.Collections.Generic.HashSet(), new System.Collections.Generic.HashSet()); + + public class BurstCapableFFmpegCapabilities() : FFmpegCapabilities( + string.Empty, + new System.Collections.Generic.HashSet(), + new System.Collections.Generic.HashSet(), + new System.Collections.Generic.HashSet(), + new System.Collections.Generic.HashSet(), + new System.Collections.Generic.HashSet { FFmpegKnownOption.ReadrateInitialBurst.Name }, + new System.Collections.Generic.HashSet()); } diff --git a/ErsatzTV.FFmpeg/Capabilities/FFmpegKnownOption.cs b/ErsatzTV.FFmpeg/Capabilities/FFmpegKnownOption.cs index 79f8ffc2f..c4a66cb89 100644 --- a/ErsatzTV.FFmpeg/Capabilities/FFmpegKnownOption.cs +++ b/ErsatzTV.FFmpeg/Capabilities/FFmpegKnownOption.cs @@ -10,7 +10,11 @@ public record FFmpegKnownOption public string Name { get; } + // ffmpeg 6.1+; lets a readrate-throttled input read flat out for an initial window + public static FFmpegKnownOption ReadrateInitialBurst => new("readrate_initial_burst"); + public static IList AllOptions => [ + ReadrateInitialBurst.Name ]; } diff --git a/ErsatzTV.FFmpeg/InputOption/ReadrateInputOption.cs b/ErsatzTV.FFmpeg/InputOption/ReadrateInputOption.cs index 216d34983..a7f4e9163 100644 --- a/ErsatzTV.FFmpeg/InputOption/ReadrateInputOption.cs +++ b/ErsatzTV.FFmpeg/InputOption/ReadrateInputOption.cs @@ -1,19 +1,37 @@ -using System.Globalization; +using System.Globalization; using ErsatzTV.FFmpeg.Environment; namespace ErsatzTV.FFmpeg.InputOption; -public class ReadrateInputOption(double readRate) : IInputOption +public class ReadrateInputOption(double readRate, Option initialBurstSeconds) : IInputOption { + public ReadrateInputOption(double readRate) + : this(readRate, Option.None) + { + } + public EnvironmentVariable[] EnvironmentVariables => []; public string[] GlobalOptions => []; - public string[] InputOptions(InputFile inputFile) => - [ - "-readrate", - readRate.ToString("0.0####", CultureInfo.InvariantCulture) - ]; + public string[] InputOptions(InputFile inputFile) + { + var result = new List + { + "-readrate", + readRate.ToString("0.0####", CultureInfo.InvariantCulture) + }; + + // burst-read this much input before the readrate throttle kicks in, so a cold start doesn't + // have to wait ~realtime for the first segment to be written (ersatztv#350) + foreach (int burst in initialBurstSeconds) + { + result.Add("-readrate_initial_burst"); + result.Add(burst.ToString(CultureInfo.InvariantCulture)); + } + + return result.ToArray(); + } public string[] FilterOptions => []; public string[] OutputOptions => []; diff --git a/ErsatzTV.FFmpeg/Pipeline/PipelineBuilderBase.cs b/ErsatzTV.FFmpeg/Pipeline/PipelineBuilderBase.cs index 30779e663..76c9e60a2 100644 --- a/ErsatzTV.FFmpeg/Pipeline/PipelineBuilderBase.cs +++ b/ErsatzTV.FFmpeg/Pipeline/PipelineBuilderBase.cs @@ -17,6 +17,11 @@ namespace ErsatzTV.FFmpeg.Pipeline; public abstract class PipelineBuilderBase : IPipelineBuilder { + // enough input to cover the first couple of HLS segments; the segmenter waits for + // ffmpeg.segmenter.initial_segment_count (default 1) of them before serving the playlist. + // an operator who raises that setting above 2 gets less of the benefit (ersatztv#350) + private const int InitialBurstSeconds = OutputFormatHls.SegmentSeconds * 2; + private readonly Option _audioInputFile; private readonly Option _concatInputFile; private readonly IFFmpegCapabilities _ffmpegCapabilities; @@ -850,8 +855,24 @@ public abstract class PipelineBuilderBase : IPipelineBuilder } double readRate = desiredState.VideoFormat == VideoFormat.Copy ? 1.0 : 1.05; - _audioInputFile.Iter(a => a.AddOption(new ReadrateInputOption(readRate))); - videoInputFile.AddOption(new ReadrateInputOption(readRate)); + + // without a burst, the readrate throttle applies from the very first read, so the first + // segment cannot be written faster than ~realtime and every start pays a multi-second wait. + // burst enough input to cover the first segments, then settle to readRate. note this is + // per ffmpeg process, i.e. per playout item, not only on the session's cold start + // (ersatztv#350) + // + // a still image is paced by the realtime filter instead, and its video input takes no + // readrate at all, so bursting there would only run the audio input ahead of the video + bool isStillImage = videoInputFile.VideoStreams.Any(s => s.StillImage); + + Option initialBurstSeconds = + !isStillImage && _ffmpegCapabilities.HasOption(FFmpegKnownOption.ReadrateInitialBurst) + ? InitialBurstSeconds + : Option.None; + + _audioInputFile.Iter(a => a.AddOption(new ReadrateInputOption(readRate, initialBurstSeconds))); + videoInputFile.AddOption(new ReadrateInputOption(readRate, initialBurstSeconds)); } protected static void SetStillImageLoop( diff --git a/docs/decisions.md b/docs/decisions.md index f59b5f3a6..6fd10cfa7 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -2521,3 +2521,46 @@ The three gated levels now share one `ChannelLogoWatermarkOptions` helper so the again — the duplication is what let the defect exist in triplicate. Covered by `WatermarkSelectorChannelLogoTests`, which pins the external-URL fix, both preserved regressions (cached local path, missing local file ignored) and the generated-fallback scope guard. +## 2026-07-20 — HLS cold start is fixed with `-readrate_initial_burst`, not by raising the work-ahead limit (#350) + +- **`-readrate` throttles from the first read, so it sets a floor on time-to-first-segment.** The + realtime playback path pins input reading to 1.05× wall clock so a channel behaves like live TV. + Since `OutputFormatHls.SegmentSeconds` is 4 and the segmenter serves the playlist only once the + first segment exists, the playlist cannot appear sooner than ~4/1.05 ≈ 3.8 s. Measured on real + media: time-to-first-playlist 5369/5344 ms with `-readrate 1.05`, 648/649 ms with an initial burst. +- **The cold-start bimodality was never about the media.** `HlsSessionWorker` grants an unthrottled + `SeekAndWorkAhead` start only while `_workAheadCount < ffmpeg.segmenter.work_ahead_limit` (prod: 1); + every concurrent tune-in falls back to the throttled path. Three concurrent tunes on prod: the one + that won the slot reached `firstGop` in 866 ms, the other two in 3845 ms and 6357 ms. This is why + the earlier rounds found no correlation with subtitle burn-in, GOP length, or source file — the + variable was slot availability, and the same channel could differ 7.2× between tunes. +- **Two earlier hypotheses are falsified, not deferred.** Accurate-seek decode-discard (the issue's + ranked #1 driver) costs 30–100 ms on real prod media, and capping `-probesize`/`-analyzeduration` + buys 20–50 ms. Neither can account for seconds. Recorded here so they are not re-proposed. +- **Burst rather than a bigger work-ahead budget.** Raising `work_ahead_limit` would fix latency by + deleting the guarantee that limit exists for — it caps how many unthrottled transcodes N viewers can + start at once. The burst is bounded (`SegmentSeconds * 2` = 8 s of input, enough for the first + segments at the default `InitialSegmentCount` of 1), after which live pacing resumes. An operator + who raises `InitialSegmentCount` above 2 gets less of the benefit; that is a deliberate trade. +- **The burst is per ffmpeg process — i.e. per playout item — not per session.** `SetRealtimeInput` + runs on every pipeline build and `HlsSessionWorker` spawns a process per item, so each item boundary + bursts too; this is *not* only the session's cold start. Two consequences, both accepted: item + transitions get the same head start (a benefit), and on a channel whose items are shorter than the + burst every item transcodes unthrottled, so the instantaneous-concurrency guarantee that + `work_ahead_limit` provides is weaker than before — weaker, not absent, because `HlsSessionWorker`'s + `transcodedBuffer <= 1min` gate still stops the loop at a 60 s buffer, leaving average CPU + unchanged. Making the burst strictly cold-start-only would mean plumbing a "first process of this + session" flag through `FFmpegState`; that complexity was not judged worth a bounded peak. +- **Still images are excluded.** Their video input is paced by the realtime *filter* and takes no + readrate at all, so a burst would only run the audio input ahead of the video for songs and offline + filler, with no cold-start gain to show for it. +- **Non-HLS realtime outputs (`TransportStream`, HLS-Direct) burst too**, since + `FFmpegPlaybackSettingsCalculator` makes them unconditionally realtime. That is untested by the + benchmark, which was segmenter-only; it is kept because the same first-read throttle delays those + clients identically, and the outer `WrapSegmenter`/`Concat` processes still pace at `readrate 1.0` + with no burst. +- **Gated on runtime capability, not on a parsed version.** `-readrate_initial_burst` needs FFmpeg + ≥ 6.1, and `FFmpegKnownOption`/`HasOption` already existed for exactly this (its `AllOptions` list + had simply been empty). Detection parses `ffmpeg -h long`, so an older binary silently keeps + today's behavior instead of failing to start — the same fail-safe posture as the other capability + gates, and cheaper to reason about than the version-string parsing in `NvidiaHardwareCapabilities`.