From 1359bb613526f48f4c88f065330c7d76f1eb9318 Mon Sep 17 00:00:00 2001 From: Timothy Date: Mon, 20 Jul 2026 23:59:04 +0200 Subject: [PATCH 1/2] fix(350): burst-read the first HLS segments so cold start isn't readrate-bound MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `-readrate 1.05` paces input reading at wall clock so a channel behaves like live TV, but it applies from the very first read. With 4s HLS segments and the segmenter waiting for the first one, the playlist could not appear sooner than ~4/1.05 = 3.8s, so every tune-in that did not win a work-ahead slot paid a multi-second wait. Add `-readrate_initial_burst` (FFmpeg >= 6.1) next to `-readrate` on the normal playback path, gated on runtime capability detection via the existing `FFmpegKnownOption`/`HasOption` machinery, whose option list had simply been empty. Measured on real prod media: time-to-first-playlist 5369/5344ms -> 648/649ms. Root cause detail: the cold-start bimodality earlier rounds could not explain was never about the media. `HlsSessionWorker` grants an unthrottled start only while `_workAheadCount < work_ahead_limit` (prod: 1), so concurrent tune-ins fall back to the throttled path. Confirmed on prod with three concurrent tunes: firstGop 866ms for the slot winner vs 3845ms and 6357ms. This also falsifies the issue's ranked #1 driver — accurate-seek decode-discard measures 30-100ms on real media, and probe caps 20-50ms; neither can account for seconds. Still images are excluded: their video input is paced by the realtime filter and takes no readrate, so a burst would only run a song's separate audio input ahead of the video. Concat/WrapSegmenter keep the unburst single-arg constructor. fixes #350 --- .../PipelineBuilderBaseTests.cs | 167 +++++++++++++++++- .../Capabilities/FFmpegKnownOption.cs | 4 + .../InputOption/ReadrateInputOption.cs | 32 +++- .../Pipeline/PipelineBuilderBase.cs | 25 ++- docs/decisions.md | 43 +++++ 5 files changed, 261 insertions(+), 10 deletions(-) diff --git a/ErsatzTV.FFmpeg.Tests/PipelineBuilderBaseTests.cs b/ErsatzTV.FFmpeg.Tests/PipelineBuilderBaseTests.cs index 64e485b4e..c224f2576 100644 --- a/ErsatzTV.FFmpeg.Tests/PipelineBuilderBaseTests.cs +++ b/ErsatzTV.FFmpeg.Tests/PipelineBuilderBaseTests.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using ErsatzTV.FFmpeg.Capabilities; using ErsatzTV.FFmpeg.Encoder; @@ -531,6 +531,162 @@ public class PipelineBuilderBaseTests "-nostdin -hide_banner -nostats -loglevel error -i /test/input/file.png -vf scale=-1:200:force_original_aspect_ratio=decrease /test/output/file.jpg"); } + [Test] + public void Realtime_Input_Should_Burst_Initial_Segments_When_Option_Is_Supported() + { + string command = BuildRealtimeCommand(new BurstCapableFFmpegCapabilities()); + + // burst covers the first two 4s segments, then the 1.05 throttle resumes (ersatztv#350) + command.ShouldContain("-readrate 1.05 -readrate_initial_burst 8 -i"); + } + + [Test] + public void Realtime_Input_Should_Not_Burst_When_Option_Is_Unsupported() + { + string command = BuildRealtimeCommand(new DefaultFFmpegCapabilities()); + + command.ShouldContain("-readrate 1.05 -i"); + command.ShouldNotContain("-readrate_initial_burst"); + } + + [Test] + public void Realtime_Input_Should_Not_Burst_A_Still_Image() + { + // a still image is paced by the realtime filter, so bursting would only run audio ahead + string command = BuildRealtimeCommand(new BurstCapableFFmpegCapabilities(), stillImage: true); + + command.ShouldNotContain("-readrate_initial_burst"); + } + + [Test] + public void Concat_Should_Never_Burst() + { + // concat reads already-written segments from the running segmenter; bursting would + // gallop through them + var concatInputFile = new ConcatInputFile("http://localhost:8080/ffmpeg/concat/1", new FrameSize(1920, 1080)); + + var builder = new SoftwarePipelineBuilder( + new BurstCapableFFmpegCapabilities(), + HardwareAccelerationMode.None, + None, + None, + None, + None, + concatInputFile, + Option.None, + "", + "", + _logger); + + FFmpegPipeline result = builder.Concat(concatInputFile, FFmpegState.Concat(false, "Some Channel")); + + string command = PrintCommand(None, None, None, concatInputFile, None, result); + + command.ShouldContain("-readrate 1.0"); + command.ShouldNotContain("-readrate_initial_burst"); + } + + private string BuildRealtimeCommand(IFFmpegCapabilities capabilities, bool stillImage = false) + { + var videoInputFile = new VideoInputFile( + "/tmp/whatever.mkv", + new List + { + new( + 0, + VideoFormat.H264, + VideoProfile.Main, + new PixelFormatYuv420P(), + ColorParams.Default, + new FrameSize(1920, 1080), + "1:1", + "16:9", + FrameRate.DefaultFrameRate, + stillImage, + ScanKind.Progressive) + }); + + var desiredState = new FrameState( + true, + false, + VideoFormat.Hevc, + VideoProfile.Main, + VideoPreset.Unset, + false, + new PixelFormatYuv420P(), + new FrameSize(1920, 1080), + new FrameSize(1920, 1080), + Option.None, + FFmpegFilterMode.HardwareIfPossible, + false, + Option.None, + 2000, + 4000, + 90_000, + false, + false); + + var ffmpegState = new FFmpegState( + false, + HardwareAccelerationMode.None, + HardwareAccelerationMode.None, + Option.None, + Option.None, + TimeSpan.FromSeconds(1), + Option.None, + false, + Option.None, + Option.None, + Option.None, + Option.None, + Option.None, + OutputFormatKind.MpegTs, + Option.None, + Option.None, + Option.None, + Option.None, + TimeSpan.Zero, + Option.None, + Option.None, + false, + false, + "clip", + false); + + // a *separate* audio input matters here: for a still image the video input takes no readrate + // at all, so only a distinct audio input can prove the burst was suppressed (this is the + // song shape — cover art plus an audio file) + var audioInputFile = new AudioInputFile( + stillImage ? "/tmp/whatever.mp3" : "/tmp/whatever.mkv", + new List { new(1, AudioFormat.Aac, 2) }, + new AudioState( + AudioFormat.Aac, + 2, + 320, + 640, + 48, + false, + AudioFilter.None, + Option.None)); + + var builder = new SoftwarePipelineBuilder( + capabilities, + HardwareAccelerationMode.None, + videoInputFile, + audioInputFile, + None, + None, + None, + Option.None, + "", + "", + _logger); + + FFmpegPipeline result = builder.Build(ffmpegState, desiredState); + + return PrintCommand(videoInputFile, audioInputFile, None, None, None, result); + } + private static string PrintCommand( Option videoInputFile, Option audioInputFile, @@ -563,4 +719,13 @@ public class PipelineBuilderBaseTests new System.Collections.Generic.HashSet(), new System.Collections.Generic.HashSet(), new System.Collections.Generic.HashSet()); + + public class BurstCapableFFmpegCapabilities() : FFmpegCapabilities( + string.Empty, + new System.Collections.Generic.HashSet(), + new System.Collections.Generic.HashSet(), + new System.Collections.Generic.HashSet(), + new System.Collections.Generic.HashSet(), + new System.Collections.Generic.HashSet { FFmpegKnownOption.ReadrateInitialBurst.Name }, + new System.Collections.Generic.HashSet()); } diff --git a/ErsatzTV.FFmpeg/Capabilities/FFmpegKnownOption.cs b/ErsatzTV.FFmpeg/Capabilities/FFmpegKnownOption.cs index 79f8ffc2f..c4a66cb89 100644 --- a/ErsatzTV.FFmpeg/Capabilities/FFmpegKnownOption.cs +++ b/ErsatzTV.FFmpeg/Capabilities/FFmpegKnownOption.cs @@ -10,7 +10,11 @@ public record FFmpegKnownOption public string Name { get; } + // ffmpeg 6.1+; lets a readrate-throttled input read flat out for an initial window + public static FFmpegKnownOption ReadrateInitialBurst => new("readrate_initial_burst"); + public static IList AllOptions => [ + ReadrateInitialBurst.Name ]; } diff --git a/ErsatzTV.FFmpeg/InputOption/ReadrateInputOption.cs b/ErsatzTV.FFmpeg/InputOption/ReadrateInputOption.cs index 216d34983..a7f4e9163 100644 --- a/ErsatzTV.FFmpeg/InputOption/ReadrateInputOption.cs +++ b/ErsatzTV.FFmpeg/InputOption/ReadrateInputOption.cs @@ -1,19 +1,37 @@ -using System.Globalization; +using System.Globalization; using ErsatzTV.FFmpeg.Environment; namespace ErsatzTV.FFmpeg.InputOption; -public class ReadrateInputOption(double readRate) : IInputOption +public class ReadrateInputOption(double readRate, Option initialBurstSeconds) : IInputOption { + public ReadrateInputOption(double readRate) + : this(readRate, Option.None) + { + } + public EnvironmentVariable[] EnvironmentVariables => []; public string[] GlobalOptions => []; - public string[] InputOptions(InputFile inputFile) => - [ - "-readrate", - readRate.ToString("0.0####", CultureInfo.InvariantCulture) - ]; + public string[] InputOptions(InputFile inputFile) + { + var result = new List + { + "-readrate", + readRate.ToString("0.0####", CultureInfo.InvariantCulture) + }; + + // burst-read this much input before the readrate throttle kicks in, so a cold start doesn't + // have to wait ~realtime for the first segment to be written (ersatztv#350) + foreach (int burst in initialBurstSeconds) + { + result.Add("-readrate_initial_burst"); + result.Add(burst.ToString(CultureInfo.InvariantCulture)); + } + + return result.ToArray(); + } public string[] FilterOptions => []; public string[] OutputOptions => []; diff --git a/ErsatzTV.FFmpeg/Pipeline/PipelineBuilderBase.cs b/ErsatzTV.FFmpeg/Pipeline/PipelineBuilderBase.cs index 30779e663..76c9e60a2 100644 --- a/ErsatzTV.FFmpeg/Pipeline/PipelineBuilderBase.cs +++ b/ErsatzTV.FFmpeg/Pipeline/PipelineBuilderBase.cs @@ -17,6 +17,11 @@ namespace ErsatzTV.FFmpeg.Pipeline; public abstract class PipelineBuilderBase : IPipelineBuilder { + // enough input to cover the first couple of HLS segments; the segmenter waits for + // ffmpeg.segmenter.initial_segment_count (default 1) of them before serving the playlist. + // an operator who raises that setting above 2 gets less of the benefit (ersatztv#350) + private const int InitialBurstSeconds = OutputFormatHls.SegmentSeconds * 2; + private readonly Option _audioInputFile; private readonly Option _concatInputFile; private readonly IFFmpegCapabilities _ffmpegCapabilities; @@ -850,8 +855,24 @@ public abstract class PipelineBuilderBase : IPipelineBuilder } double readRate = desiredState.VideoFormat == VideoFormat.Copy ? 1.0 : 1.05; - _audioInputFile.Iter(a => a.AddOption(new ReadrateInputOption(readRate))); - videoInputFile.AddOption(new ReadrateInputOption(readRate)); + + // without a burst, the readrate throttle applies from the very first read, so the first + // segment cannot be written faster than ~realtime and every start pays a multi-second wait. + // burst enough input to cover the first segments, then settle to readRate. note this is + // per ffmpeg process, i.e. per playout item, not only on the session's cold start + // (ersatztv#350) + // + // a still image is paced by the realtime filter instead, and its video input takes no + // readrate at all, so bursting there would only run the audio input ahead of the video + bool isStillImage = videoInputFile.VideoStreams.Any(s => s.StillImage); + + Option initialBurstSeconds = + !isStillImage && _ffmpegCapabilities.HasOption(FFmpegKnownOption.ReadrateInitialBurst) + ? InitialBurstSeconds + : Option.None; + + _audioInputFile.Iter(a => a.AddOption(new ReadrateInputOption(readRate, initialBurstSeconds))); + videoInputFile.AddOption(new ReadrateInputOption(readRate, initialBurstSeconds)); } protected static void SetStillImageLoop( diff --git a/docs/decisions.md b/docs/decisions.md index f59b5f3a6..6fd10cfa7 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -2521,3 +2521,46 @@ The three gated levels now share one `ChannelLogoWatermarkOptions` helper so the again — the duplication is what let the defect exist in triplicate. Covered by `WatermarkSelectorChannelLogoTests`, which pins the external-URL fix, both preserved regressions (cached local path, missing local file ignored) and the generated-fallback scope guard. +## 2026-07-20 — HLS cold start is fixed with `-readrate_initial_burst`, not by raising the work-ahead limit (#350) + +- **`-readrate` throttles from the first read, so it sets a floor on time-to-first-segment.** The + realtime playback path pins input reading to 1.05× wall clock so a channel behaves like live TV. + Since `OutputFormatHls.SegmentSeconds` is 4 and the segmenter serves the playlist only once the + first segment exists, the playlist cannot appear sooner than ~4/1.05 ≈ 3.8 s. Measured on real + media: time-to-first-playlist 5369/5344 ms with `-readrate 1.05`, 648/649 ms with an initial burst. +- **The cold-start bimodality was never about the media.** `HlsSessionWorker` grants an unthrottled + `SeekAndWorkAhead` start only while `_workAheadCount < ffmpeg.segmenter.work_ahead_limit` (prod: 1); + every concurrent tune-in falls back to the throttled path. Three concurrent tunes on prod: the one + that won the slot reached `firstGop` in 866 ms, the other two in 3845 ms and 6357 ms. This is why + the earlier rounds found no correlation with subtitle burn-in, GOP length, or source file — the + variable was slot availability, and the same channel could differ 7.2× between tunes. +- **Two earlier hypotheses are falsified, not deferred.** Accurate-seek decode-discard (the issue's + ranked #1 driver) costs 30–100 ms on real prod media, and capping `-probesize`/`-analyzeduration` + buys 20–50 ms. Neither can account for seconds. Recorded here so they are not re-proposed. +- **Burst rather than a bigger work-ahead budget.** Raising `work_ahead_limit` would fix latency by + deleting the guarantee that limit exists for — it caps how many unthrottled transcodes N viewers can + start at once. The burst is bounded (`SegmentSeconds * 2` = 8 s of input, enough for the first + segments at the default `InitialSegmentCount` of 1), after which live pacing resumes. An operator + who raises `InitialSegmentCount` above 2 gets less of the benefit; that is a deliberate trade. +- **The burst is per ffmpeg process — i.e. per playout item — not per session.** `SetRealtimeInput` + runs on every pipeline build and `HlsSessionWorker` spawns a process per item, so each item boundary + bursts too; this is *not* only the session's cold start. Two consequences, both accepted: item + transitions get the same head start (a benefit), and on a channel whose items are shorter than the + burst every item transcodes unthrottled, so the instantaneous-concurrency guarantee that + `work_ahead_limit` provides is weaker than before — weaker, not absent, because `HlsSessionWorker`'s + `transcodedBuffer <= 1min` gate still stops the loop at a 60 s buffer, leaving average CPU + unchanged. Making the burst strictly cold-start-only would mean plumbing a "first process of this + session" flag through `FFmpegState`; that complexity was not judged worth a bounded peak. +- **Still images are excluded.** Their video input is paced by the realtime *filter* and takes no + readrate at all, so a burst would only run the audio input ahead of the video for songs and offline + filler, with no cold-start gain to show for it. +- **Non-HLS realtime outputs (`TransportStream`, HLS-Direct) burst too**, since + `FFmpegPlaybackSettingsCalculator` makes them unconditionally realtime. That is untested by the + benchmark, which was segmenter-only; it is kept because the same first-read throttle delays those + clients identically, and the outer `WrapSegmenter`/`Concat` processes still pace at `readrate 1.0` + with no burst. +- **Gated on runtime capability, not on a parsed version.** `-readrate_initial_burst` needs FFmpeg + ≥ 6.1, and `FFmpegKnownOption`/`HasOption` already existed for exactly this (its `AllOptions` list + had simply been empty). Detection parses `ffmpeg -h long`, so an older binary silently keeps + today's behavior instead of failing to start — the same fail-safe posture as the other capability + gates, and cheaper to reason about than the version-string parsing in `NvidiaHardwareCapabilities`. -- 2.47.3 From 0e3b0ca309e4c93530e599075b6ff571fb72cbe9 Mon Sep 17 00:00:00 2001 From: Timothy Date: Tue, 21 Jul 2026 00:07:35 +0200 Subject: [PATCH 2/2] test(350): harden the burst tests against silent regressions Re-review of the fix commit flagged two ways the new tests could pass vacuously in future: the burst assertion was a bare substring (satisfiable by any input carrying the option) and the still-image test asserted only an absence. Anchor the first on the input path plus an occurrence count, and give the second a positive anchor. --- ErsatzTV.FFmpeg.Tests/PipelineBuilderBaseTests.cs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/ErsatzTV.FFmpeg.Tests/PipelineBuilderBaseTests.cs b/ErsatzTV.FFmpeg.Tests/PipelineBuilderBaseTests.cs index c224f2576..53f4114e9 100644 --- a/ErsatzTV.FFmpeg.Tests/PipelineBuilderBaseTests.cs +++ b/ErsatzTV.FFmpeg.Tests/PipelineBuilderBaseTests.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Text.RegularExpressions; using ErsatzTV.FFmpeg.Capabilities; using ErsatzTV.FFmpeg.Encoder; using ErsatzTV.FFmpeg.Format; @@ -536,8 +537,11 @@ public class PipelineBuilderBaseTests { string command = BuildRealtimeCommand(new BurstCapableFFmpegCapabilities()); - // burst covers the first two 4s segments, then the 1.05 throttle resumes (ersatztv#350) - command.ShouldContain("-readrate 1.05 -readrate_initial_burst 8 -i"); + // burst covers the first two 4s segments, then the 1.05 throttle resumes (ersatztv#350). + // anchor on the input path so this can't be satisfied by some other input carrying the + // burst; audio and video share one file here, so exactly one input is expected + command.ShouldContain("-readrate 1.05 -readrate_initial_burst 8 -i /tmp/whatever.mkv"); + Regex.Matches(command, "-readrate_initial_burst 8").Count.ShouldBe(1); } [Test] @@ -555,6 +559,9 @@ public class PipelineBuilderBaseTests // a still image is paced by the realtime filter, so bursting would only run audio ahead string command = BuildRealtimeCommand(new BurstCapableFFmpegCapabilities(), stillImage: true); + // the positive anchor keeps this from passing vacuously if the helper ever stops + // producing a realtime audio input at all + command.ShouldContain("-readrate 1.05"); command.ShouldNotContain("-readrate_initial_burst"); } -- 2.47.3