fix(350): burst-read the first HLS segments so cold start isn't readrate-bound #516

Merged
timothy merged 2 commits from fix/350-coldstart-seek into main 2026-07-21 00:33:40 +02:00
5 changed files with 268 additions and 10 deletions
@@ -1,5 +1,6 @@
using System;
using System;
using System.Collections.Generic;
using System.Text.RegularExpressions;
using ErsatzTV.FFmpeg.Capabilities;
using ErsatzTV.FFmpeg.Encoder;
using ErsatzTV.FFmpeg.Format;
@@ -531,6 +532,168 @@ public class PipelineBuilderBaseTests
"-nostdin -hide_banner -nostats -loglevel error -i /test/input/file.png -vf scale=-1:200:force_original_aspect_ratio=decrease /test/output/file.jpg");
}
[Test]
public void Realtime_Input_Should_Burst_Initial_Segments_When_Option_Is_Supported()
{
string command = BuildRealtimeCommand(new BurstCapableFFmpegCapabilities());
// burst covers the first two 4s segments, then the 1.05 throttle resumes (ersatztv#350).
// anchor on the input path so this can't be satisfied by some other input carrying the
// burst; audio and video share one file here, so exactly one input is expected
command.ShouldContain("-readrate 1.05 -readrate_initial_burst 8 -i /tmp/whatever.mkv");
Regex.Matches(command, "-readrate_initial_burst 8").Count.ShouldBe(1);
}
[Test]
public void Realtime_Input_Should_Not_Burst_When_Option_Is_Unsupported()
{
string command = BuildRealtimeCommand(new DefaultFFmpegCapabilities());
command.ShouldContain("-readrate 1.05 -i");
command.ShouldNotContain("-readrate_initial_burst");
}
[Test]
public void Realtime_Input_Should_Not_Burst_A_Still_Image()
{
// a still image is paced by the realtime filter, so bursting would only run audio ahead
string command = BuildRealtimeCommand(new BurstCapableFFmpegCapabilities(), stillImage: true);
// the positive anchor keeps this from passing vacuously if the helper ever stops
// producing a realtime audio input at all
command.ShouldContain("-readrate 1.05");
command.ShouldNotContain("-readrate_initial_burst");
}
[Test]
public void Concat_Should_Never_Burst()
{
// concat reads already-written segments from the running segmenter; bursting would
// gallop through them
var concatInputFile = new ConcatInputFile("http://localhost:8080/ffmpeg/concat/1", new FrameSize(1920, 1080));
var builder = new SoftwarePipelineBuilder(
new BurstCapableFFmpegCapabilities(),
HardwareAccelerationMode.None,
None,
None,
None,
None,
concatInputFile,
Option<GraphicsEngineInput>.None,
"",
"",
_logger);
FFmpegPipeline result = builder.Concat(concatInputFile, FFmpegState.Concat(false, "Some Channel"));
string command = PrintCommand(None, None, None, concatInputFile, None, result);
command.ShouldContain("-readrate 1.0");
command.ShouldNotContain("-readrate_initial_burst");
}
private string BuildRealtimeCommand(IFFmpegCapabilities capabilities, bool stillImage = false)
{
var videoInputFile = new VideoInputFile(
"/tmp/whatever.mkv",
new List<VideoStream>
{
new(
0,
VideoFormat.H264,
VideoProfile.Main,
new PixelFormatYuv420P(),
ColorParams.Default,
new FrameSize(1920, 1080),
"1:1",
"16:9",
FrameRate.DefaultFrameRate,
stillImage,
ScanKind.Progressive)
});
var desiredState = new FrameState(
true,
false,
VideoFormat.Hevc,
VideoProfile.Main,
VideoPreset.Unset,
false,
new PixelFormatYuv420P(),
new FrameSize(1920, 1080),
new FrameSize(1920, 1080),
Option<FrameSize>.None,
FFmpegFilterMode.HardwareIfPossible,
false,
Option<FrameRate>.None,
2000,
4000,
90_000,
false,
false);
var ffmpegState = new FFmpegState(
false,
HardwareAccelerationMode.None,
HardwareAccelerationMode.None,
Option<string>.None,
Option<string>.None,
TimeSpan.FromSeconds(1),
Option<TimeSpan>.None,
false,
Option<string>.None,
Option<string>.None,
Option<string>.None,
Option<string>.None,
Option<string>.None,
OutputFormatKind.MpegTs,
Option<string>.None,
Option<string>.None,
Option<string>.None,
Option<string>.None,
TimeSpan.Zero,
Option<int>.None,
Option<int>.None,
false,
false,
"clip",
false);
// a *separate* audio input matters here: for a still image the video input takes no readrate
// at all, so only a distinct audio input can prove the burst was suppressed (this is the
// song shape — cover art plus an audio file)
var audioInputFile = new AudioInputFile(
stillImage ? "/tmp/whatever.mp3" : "/tmp/whatever.mkv",
new List<AudioStream> { new(1, AudioFormat.Aac, 2) },
new AudioState(
AudioFormat.Aac,
2,
320,
640,
48,
false,
AudioFilter.None,
Option<double>.None));
var builder = new SoftwarePipelineBuilder(
capabilities,
HardwareAccelerationMode.None,
videoInputFile,
audioInputFile,
None,
None,
None,
Option<GraphicsEngineInput>.None,
"",
"",
_logger);
FFmpegPipeline result = builder.Build(ffmpegState, desiredState);
return PrintCommand(videoInputFile, audioInputFile, None, None, None, result);
}
private static string PrintCommand(
Option<VideoInputFile> videoInputFile,
Option<AudioInputFile> audioInputFile,
@@ -563,4 +726,13 @@ public class PipelineBuilderBaseTests
new System.Collections.Generic.HashSet<string>(),
new System.Collections.Generic.HashSet<string>(),
new System.Collections.Generic.HashSet<string>());
public class BurstCapableFFmpegCapabilities() : FFmpegCapabilities(
string.Empty,
new System.Collections.Generic.HashSet<string>(),
new System.Collections.Generic.HashSet<string>(),
new System.Collections.Generic.HashSet<string>(),
new System.Collections.Generic.HashSet<string>(),
new System.Collections.Generic.HashSet<string> { FFmpegKnownOption.ReadrateInitialBurst.Name },
new System.Collections.Generic.HashSet<string>());
}
@@ -10,7 +10,11 @@ public record FFmpegKnownOption
public string Name { get; }
// ffmpeg 6.1+; lets a readrate-throttled input read flat out for an initial window
public static FFmpegKnownOption ReadrateInitialBurst => new("readrate_initial_burst");
public static IList<string> AllOptions =>
[
ReadrateInitialBurst.Name
];
}
@@ -1,19 +1,37 @@
using System.Globalization;
using System.Globalization;
using ErsatzTV.FFmpeg.Environment;
namespace ErsatzTV.FFmpeg.InputOption;
public class ReadrateInputOption(double readRate) : IInputOption
public class ReadrateInputOption(double readRate, Option<int> initialBurstSeconds) : IInputOption
{
public ReadrateInputOption(double readRate)
: this(readRate, Option<int>.None)
{
}
public EnvironmentVariable[] EnvironmentVariables => [];
public string[] GlobalOptions => [];
public string[] InputOptions(InputFile inputFile) =>
[
"-readrate",
readRate.ToString("0.0####", CultureInfo.InvariantCulture)
];
public string[] InputOptions(InputFile inputFile)
{
var result = new List<string>
{
"-readrate",
readRate.ToString("0.0####", CultureInfo.InvariantCulture)
};
// burst-read this much input before the readrate throttle kicks in, so a cold start doesn't
// have to wait ~realtime for the first segment to be written (ersatztv#350)
foreach (int burst in initialBurstSeconds)
{
result.Add("-readrate_initial_burst");
result.Add(burst.ToString(CultureInfo.InvariantCulture));
}
return result.ToArray();
}
public string[] FilterOptions => [];
public string[] OutputOptions => [];
@@ -17,6 +17,11 @@ namespace ErsatzTV.FFmpeg.Pipeline;
public abstract class PipelineBuilderBase : IPipelineBuilder
{
// enough input to cover the first couple of HLS segments; the segmenter waits for
// ffmpeg.segmenter.initial_segment_count (default 1) of them before serving the playlist.
// an operator who raises that setting above 2 gets less of the benefit (ersatztv#350)
private const int InitialBurstSeconds = OutputFormatHls.SegmentSeconds * 2;
private readonly Option<AudioInputFile> _audioInputFile;
private readonly Option<ConcatInputFile> _concatInputFile;
private readonly IFFmpegCapabilities _ffmpegCapabilities;
@@ -850,8 +855,24 @@ public abstract class PipelineBuilderBase : IPipelineBuilder
}
double readRate = desiredState.VideoFormat == VideoFormat.Copy ? 1.0 : 1.05;
_audioInputFile.Iter(a => a.AddOption(new ReadrateInputOption(readRate)));
videoInputFile.AddOption(new ReadrateInputOption(readRate));
// without a burst, the readrate throttle applies from the very first read, so the first
// segment cannot be written faster than ~realtime and every start pays a multi-second wait.
// burst enough input to cover the first segments, then settle to readRate. note this is
// per ffmpeg process, i.e. per playout item, not only on the session's cold start
// (ersatztv#350)
//
// a still image is paced by the realtime filter instead, and its video input takes no
// readrate at all, so bursting there would only run the audio input ahead of the video
bool isStillImage = videoInputFile.VideoStreams.Any(s => s.StillImage);
Option<int> initialBurstSeconds =
!isStillImage && _ffmpegCapabilities.HasOption(FFmpegKnownOption.ReadrateInitialBurst)
? InitialBurstSeconds
: Option<int>.None;
_audioInputFile.Iter(a => a.AddOption(new ReadrateInputOption(readRate, initialBurstSeconds)));
videoInputFile.AddOption(new ReadrateInputOption(readRate, initialBurstSeconds));
}
protected static void SetStillImageLoop(
+43
View File
@@ -2521,3 +2521,46 @@ The three gated levels now share one `ChannelLogoWatermarkOptions` helper so the
again — the duplication is what let the defect exist in triplicate. Covered by
`WatermarkSelectorChannelLogoTests`, which pins the external-URL fix, both preserved regressions
(cached local path, missing local file ignored) and the generated-fallback scope guard.
## 2026-07-20 — HLS cold start is fixed with `-readrate_initial_burst`, not by raising the work-ahead limit (#350)
- **`-readrate` throttles from the first read, so it sets a floor on time-to-first-segment.** The
realtime playback path pins input reading to 1.05× wall clock so a channel behaves like live TV.
Since `OutputFormatHls.SegmentSeconds` is 4 and the segmenter serves the playlist only once the
first segment exists, the playlist cannot appear sooner than ~4/1.05 ≈ 3.8 s. Measured on real
media: time-to-first-playlist 5369/5344 ms with `-readrate 1.05`, 648/649 ms with an initial burst.
- **The cold-start bimodality was never about the media.** `HlsSessionWorker` grants an unthrottled
`SeekAndWorkAhead` start only while `_workAheadCount < ffmpeg.segmenter.work_ahead_limit` (prod: 1);
every concurrent tune-in falls back to the throttled path. Three concurrent tunes on prod: the one
that won the slot reached `firstGop` in 866 ms, the other two in 3845 ms and 6357 ms. This is why
the earlier rounds found no correlation with subtitle burn-in, GOP length, or source file — the
variable was slot availability, and the same channel could differ 7.2× between tunes.
- **Two earlier hypotheses are falsified, not deferred.** Accurate-seek decode-discard (the issue's
ranked #1 driver) costs 30–100 ms on real prod media, and capping `-probesize`/`-analyzeduration`
buys 20–50 ms. Neither can account for seconds. Recorded here so they are not re-proposed.
- **Burst rather than a bigger work-ahead budget.** Raising `work_ahead_limit` would fix latency by
deleting the guarantee that limit exists for — it caps how many unthrottled transcodes N viewers can
start at once. The burst is bounded (`SegmentSeconds * 2` = 8 s of input, enough for the first
segments at the default `InitialSegmentCount` of 1), after which live pacing resumes. An operator
who raises `InitialSegmentCount` above 2 gets less of the benefit; that is a deliberate trade.
- **The burst is per ffmpeg process — i.e. per playout item — not per session.** `SetRealtimeInput`
runs on every pipeline build and `HlsSessionWorker` spawns a process per item, so each item boundary
bursts too; this is *not* only the session's cold start. Two consequences, both accepted: item
transitions get the same head start (a benefit), and on a channel whose items are shorter than the
burst every item transcodes unthrottled, so the instantaneous-concurrency guarantee that
`work_ahead_limit` provides is weaker than before — weaker, not absent, because `HlsSessionWorker`'s
`transcodedBuffer <= 1min` gate still stops the loop at a 60 s buffer, leaving average CPU
unchanged. Making the burst strictly cold-start-only would mean plumbing a "first process of this
session" flag through `FFmpegState`; that complexity was not judged worth a bounded peak.
- **Still images are excluded.** Their video input is paced by the realtime *filter* and takes no
readrate at all, so a burst would only run the audio input ahead of the video for songs and offline
filler, with no cold-start gain to show for it.
- **Non-HLS realtime outputs (`TransportStream`, HLS-Direct) burst too**, since
`FFmpegPlaybackSettingsCalculator` makes them unconditionally realtime. That is untested by the
benchmark, which was segmenter-only; it is kept because the same first-read throttle delays those
clients identically, and the outer `WrapSegmenter`/`Concat` processes still pace at `readrate 1.0`
with no burst.
- **Gated on runtime capability, not on a parsed version.** `-readrate_initial_burst` needs FFmpeg
≥ 6.1, and `FFmpegKnownOption`/`HasOption` already existed for exactly this (its `AllOptions` list
had simply been empty). Detection parses `ffmpeg -h long`, so an older binary silently keeps
today's behavior instead of failing to start — the same fail-safe posture as the other capability
gates, and cheaper to reason about than the version-string parsing in `NvidiaHardwareCapabilities`.