fix(350): burst-read the first HLS segments so cold start isn't readrate-bound #516
@@ -1,5 +1,6 @@
|
||||
using System;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text.RegularExpressions;
|
||||
using ErsatzTV.FFmpeg.Capabilities;
|
||||
using ErsatzTV.FFmpeg.Encoder;
|
||||
using ErsatzTV.FFmpeg.Format;
|
||||
@@ -531,6 +532,168 @@ public class PipelineBuilderBaseTests
|
||||
"-nostdin -hide_banner -nostats -loglevel error -i /test/input/file.png -vf scale=-1:200:force_original_aspect_ratio=decrease /test/output/file.jpg");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void Realtime_Input_Should_Burst_Initial_Segments_When_Option_Is_Supported()
|
||||
{
|
||||
string command = BuildRealtimeCommand(new BurstCapableFFmpegCapabilities());
|
||||
|
||||
// burst covers the first two 4s segments, then the 1.05 throttle resumes (ersatztv#350).
|
||||
// anchor on the input path so this can't be satisfied by some other input carrying the
|
||||
// burst; audio and video share one file here, so exactly one input is expected
|
||||
command.ShouldContain("-readrate 1.05 -readrate_initial_burst 8 -i /tmp/whatever.mkv");
|
||||
Regex.Matches(command, "-readrate_initial_burst 8").Count.ShouldBe(1);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void Realtime_Input_Should_Not_Burst_When_Option_Is_Unsupported()
|
||||
{
|
||||
string command = BuildRealtimeCommand(new DefaultFFmpegCapabilities());
|
||||
|
||||
command.ShouldContain("-readrate 1.05 -i");
|
||||
command.ShouldNotContain("-readrate_initial_burst");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void Realtime_Input_Should_Not_Burst_A_Still_Image()
|
||||
{
|
||||
// a still image is paced by the realtime filter, so bursting would only run audio ahead
|
||||
string command = BuildRealtimeCommand(new BurstCapableFFmpegCapabilities(), stillImage: true);
|
||||
|
||||
// the positive anchor keeps this from passing vacuously if the helper ever stops
|
||||
// producing a realtime audio input at all
|
||||
command.ShouldContain("-readrate 1.05");
|
||||
command.ShouldNotContain("-readrate_initial_burst");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void Concat_Should_Never_Burst()
|
||||
{
|
||||
// concat reads already-written segments from the running segmenter; bursting would
|
||||
// gallop through them
|
||||
var concatInputFile = new ConcatInputFile("http://localhost:8080/ffmpeg/concat/1", new FrameSize(1920, 1080));
|
||||
|
||||
var builder = new SoftwarePipelineBuilder(
|
||||
new BurstCapableFFmpegCapabilities(),
|
||||
HardwareAccelerationMode.None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
concatInputFile,
|
||||
Option<GraphicsEngineInput>.None,
|
||||
"",
|
||||
"",
|
||||
_logger);
|
||||
|
||||
FFmpegPipeline result = builder.Concat(concatInputFile, FFmpegState.Concat(false, "Some Channel"));
|
||||
|
||||
string command = PrintCommand(None, None, None, concatInputFile, None, result);
|
||||
|
||||
command.ShouldContain("-readrate 1.0");
|
||||
command.ShouldNotContain("-readrate_initial_burst");
|
||||
}
|
||||
|
||||
private string BuildRealtimeCommand(IFFmpegCapabilities capabilities, bool stillImage = false)
|
||||
{
|
||||
var videoInputFile = new VideoInputFile(
|
||||
"/tmp/whatever.mkv",
|
||||
new List<VideoStream>
|
||||
{
|
||||
new(
|
||||
0,
|
||||
VideoFormat.H264,
|
||||
VideoProfile.Main,
|
||||
new PixelFormatYuv420P(),
|
||||
ColorParams.Default,
|
||||
new FrameSize(1920, 1080),
|
||||
"1:1",
|
||||
"16:9",
|
||||
FrameRate.DefaultFrameRate,
|
||||
stillImage,
|
||||
ScanKind.Progressive)
|
||||
});
|
||||
|
||||
var desiredState = new FrameState(
|
||||
true,
|
||||
false,
|
||||
VideoFormat.Hevc,
|
||||
VideoProfile.Main,
|
||||
VideoPreset.Unset,
|
||||
false,
|
||||
new PixelFormatYuv420P(),
|
||||
new FrameSize(1920, 1080),
|
||||
new FrameSize(1920, 1080),
|
||||
Option<FrameSize>.None,
|
||||
FFmpegFilterMode.HardwareIfPossible,
|
||||
false,
|
||||
Option<FrameRate>.None,
|
||||
2000,
|
||||
4000,
|
||||
90_000,
|
||||
false,
|
||||
false);
|
||||
|
||||
var ffmpegState = new FFmpegState(
|
||||
false,
|
||||
HardwareAccelerationMode.None,
|
||||
HardwareAccelerationMode.None,
|
||||
Option<string>.None,
|
||||
Option<string>.None,
|
||||
TimeSpan.FromSeconds(1),
|
||||
Option<TimeSpan>.None,
|
||||
false,
|
||||
Option<string>.None,
|
||||
Option<string>.None,
|
||||
Option<string>.None,
|
||||
Option<string>.None,
|
||||
Option<string>.None,
|
||||
OutputFormatKind.MpegTs,
|
||||
Option<string>.None,
|
||||
Option<string>.None,
|
||||
Option<string>.None,
|
||||
Option<string>.None,
|
||||
TimeSpan.Zero,
|
||||
Option<int>.None,
|
||||
Option<int>.None,
|
||||
false,
|
||||
false,
|
||||
"clip",
|
||||
false);
|
||||
|
||||
// a *separate* audio input matters here: for a still image the video input takes no readrate
|
||||
// at all, so only a distinct audio input can prove the burst was suppressed (this is the
|
||||
// song shape — cover art plus an audio file)
|
||||
var audioInputFile = new AudioInputFile(
|
||||
stillImage ? "/tmp/whatever.mp3" : "/tmp/whatever.mkv",
|
||||
new List<AudioStream> { new(1, AudioFormat.Aac, 2) },
|
||||
new AudioState(
|
||||
AudioFormat.Aac,
|
||||
2,
|
||||
320,
|
||||
640,
|
||||
48,
|
||||
false,
|
||||
AudioFilter.None,
|
||||
Option<double>.None));
|
||||
|
||||
var builder = new SoftwarePipelineBuilder(
|
||||
capabilities,
|
||||
HardwareAccelerationMode.None,
|
||||
videoInputFile,
|
||||
audioInputFile,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
Option<GraphicsEngineInput>.None,
|
||||
"",
|
||||
"",
|
||||
_logger);
|
||||
|
||||
FFmpegPipeline result = builder.Build(ffmpegState, desiredState);
|
||||
|
||||
return PrintCommand(videoInputFile, audioInputFile, None, None, None, result);
|
||||
}
|
||||
|
||||
private static string PrintCommand(
|
||||
Option<VideoInputFile> videoInputFile,
|
||||
Option<AudioInputFile> audioInputFile,
|
||||
@@ -563,4 +726,13 @@ public class PipelineBuilderBaseTests
|
||||
new System.Collections.Generic.HashSet<string>(),
|
||||
new System.Collections.Generic.HashSet<string>(),
|
||||
new System.Collections.Generic.HashSet<string>());
|
||||
|
||||
public class BurstCapableFFmpegCapabilities() : FFmpegCapabilities(
|
||||
string.Empty,
|
||||
new System.Collections.Generic.HashSet<string>(),
|
||||
new System.Collections.Generic.HashSet<string>(),
|
||||
new System.Collections.Generic.HashSet<string>(),
|
||||
new System.Collections.Generic.HashSet<string>(),
|
||||
new System.Collections.Generic.HashSet<string> { FFmpegKnownOption.ReadrateInitialBurst.Name },
|
||||
new System.Collections.Generic.HashSet<string>());
|
||||
}
|
||||
|
||||
@@ -10,7 +10,11 @@ public record FFmpegKnownOption
|
||||
|
||||
public string Name { get; }
|
||||
|
||||
// ffmpeg 6.1+; lets a readrate-throttled input read flat out for an initial window
|
||||
public static FFmpegKnownOption ReadrateInitialBurst => new("readrate_initial_burst");
|
||||
|
||||
public static IList<string> AllOptions =>
|
||||
[
|
||||
ReadrateInitialBurst.Name
|
||||
];
|
||||
}
|
||||
|
||||
@@ -1,19 +1,37 @@
|
||||
using System.Globalization;
|
||||
using System.Globalization;
|
||||
using ErsatzTV.FFmpeg.Environment;
|
||||
|
||||
namespace ErsatzTV.FFmpeg.InputOption;
|
||||
|
||||
public class ReadrateInputOption(double readRate) : IInputOption
|
||||
public class ReadrateInputOption(double readRate, Option<int> initialBurstSeconds) : IInputOption
|
||||
{
|
||||
public ReadrateInputOption(double readRate)
|
||||
: this(readRate, Option<int>.None)
|
||||
{
|
||||
}
|
||||
|
||||
public EnvironmentVariable[] EnvironmentVariables => [];
|
||||
|
||||
public string[] GlobalOptions => [];
|
||||
|
||||
public string[] InputOptions(InputFile inputFile) =>
|
||||
[
|
||||
"-readrate",
|
||||
readRate.ToString("0.0####", CultureInfo.InvariantCulture)
|
||||
];
|
||||
public string[] InputOptions(InputFile inputFile)
|
||||
{
|
||||
var result = new List<string>
|
||||
{
|
||||
"-readrate",
|
||||
readRate.ToString("0.0####", CultureInfo.InvariantCulture)
|
||||
};
|
||||
|
||||
// burst-read this much input before the readrate throttle kicks in, so a cold start doesn't
|
||||
// have to wait ~realtime for the first segment to be written (ersatztv#350)
|
||||
foreach (int burst in initialBurstSeconds)
|
||||
{
|
||||
result.Add("-readrate_initial_burst");
|
||||
result.Add(burst.ToString(CultureInfo.InvariantCulture));
|
||||
}
|
||||
|
||||
return result.ToArray();
|
||||
}
|
||||
|
||||
public string[] FilterOptions => [];
|
||||
public string[] OutputOptions => [];
|
||||
|
||||
@@ -17,6 +17,11 @@ namespace ErsatzTV.FFmpeg.Pipeline;
|
||||
|
||||
public abstract class PipelineBuilderBase : IPipelineBuilder
|
||||
{
|
||||
// enough input to cover the first couple of HLS segments; the segmenter waits for
|
||||
// ffmpeg.segmenter.initial_segment_count (default 1) of them before serving the playlist.
|
||||
// an operator who raises that setting above 2 gets less of the benefit (ersatztv#350)
|
||||
private const int InitialBurstSeconds = OutputFormatHls.SegmentSeconds * 2;
|
||||
|
||||
private readonly Option<AudioInputFile> _audioInputFile;
|
||||
private readonly Option<ConcatInputFile> _concatInputFile;
|
||||
private readonly IFFmpegCapabilities _ffmpegCapabilities;
|
||||
@@ -850,8 +855,24 @@ public abstract class PipelineBuilderBase : IPipelineBuilder
|
||||
}
|
||||
|
||||
double readRate = desiredState.VideoFormat == VideoFormat.Copy ? 1.0 : 1.05;
|
||||
_audioInputFile.Iter(a => a.AddOption(new ReadrateInputOption(readRate)));
|
||||
videoInputFile.AddOption(new ReadrateInputOption(readRate));
|
||||
|
||||
// without a burst, the readrate throttle applies from the very first read, so the first
|
||||
// segment cannot be written faster than ~realtime and every start pays a multi-second wait.
|
||||
// burst enough input to cover the first segments, then settle to readRate. note this is
|
||||
// per ffmpeg process, i.e. per playout item, not only on the session's cold start
|
||||
// (ersatztv#350)
|
||||
//
|
||||
// a still image is paced by the realtime filter instead, and its video input takes no
|
||||
// readrate at all, so bursting there would only run the audio input ahead of the video
|
||||
bool isStillImage = videoInputFile.VideoStreams.Any(s => s.StillImage);
|
||||
|
||||
Option<int> initialBurstSeconds =
|
||||
!isStillImage && _ffmpegCapabilities.HasOption(FFmpegKnownOption.ReadrateInitialBurst)
|
||||
? InitialBurstSeconds
|
||||
: Option<int>.None;
|
||||
|
||||
_audioInputFile.Iter(a => a.AddOption(new ReadrateInputOption(readRate, initialBurstSeconds)));
|
||||
videoInputFile.AddOption(new ReadrateInputOption(readRate, initialBurstSeconds));
|
||||
}
|
||||
|
||||
protected static void SetStillImageLoop(
|
||||
|
||||
@@ -2521,3 +2521,46 @@ The three gated levels now share one `ChannelLogoWatermarkOptions` helper so the
|
||||
again — the duplication is what let the defect exist in triplicate. Covered by
|
||||
`WatermarkSelectorChannelLogoTests`, which pins the external-URL fix, both preserved regressions
|
||||
(cached local path, missing local file ignored) and the generated-fallback scope guard.
|
||||
## 2026-07-20 — HLS cold start is fixed with `-readrate_initial_burst`, not by raising the work-ahead limit (#350)
|
||||
|
||||
- **`-readrate` throttles from the first read, so it sets a floor on time-to-first-segment.** The
|
||||
realtime playback path pins input reading to 1.05× wall clock so a channel behaves like live TV.
|
||||
Since `OutputFormatHls.SegmentSeconds` is 4 and the segmenter serves the playlist only once the
|
||||
first segment exists, the playlist cannot appear sooner than ~4/1.05 ≈ 3.8 s. Measured on real
|
||||
media: time-to-first-playlist 5369/5344 ms with `-readrate 1.05`, 648/649 ms with an initial burst.
|
||||
- **The cold-start bimodality was never about the media.** `HlsSessionWorker` grants an unthrottled
|
||||
`SeekAndWorkAhead` start only while `_workAheadCount < ffmpeg.segmenter.work_ahead_limit` (prod: 1);
|
||||
every concurrent tune-in falls back to the throttled path. Three concurrent tunes on prod: the one
|
||||
that won the slot reached `firstGop` in 866 ms, the other two in 3845 ms and 6357 ms. This is why
|
||||
the earlier rounds found no correlation with subtitle burn-in, GOP length, or source file — the
|
||||
variable was slot availability, and the same channel could differ 7.2× between tunes.
|
||||
- **Two earlier hypotheses are falsified, not deferred.** Accurate-seek decode-discard (the issue's
|
||||
ranked #1 driver) costs 30–100 ms on real prod media, and capping `-probesize`/`-analyzeduration`
|
||||
buys 20–50 ms. Neither can account for seconds. Recorded here so they are not re-proposed.
|
||||
- **Burst rather than a bigger work-ahead budget.** Raising `work_ahead_limit` would fix latency by
|
||||
deleting the guarantee that limit exists for — it caps how many unthrottled transcodes N viewers can
|
||||
start at once. The burst is bounded (`SegmentSeconds * 2` = 8 s of input, enough for the first
|
||||
segments at the default `InitialSegmentCount` of 1), after which live pacing resumes. An operator
|
||||
who raises `InitialSegmentCount` above 2 gets less of the benefit; that is a deliberate trade.
|
||||
- **The burst is per ffmpeg process — i.e. per playout item — not per session.** `SetRealtimeInput`
|
||||
runs on every pipeline build and `HlsSessionWorker` spawns a process per item, so each item boundary
|
||||
bursts too; this is *not* only the session's cold start. Two consequences, both accepted: item
|
||||
transitions get the same head start (a benefit), and on a channel whose items are shorter than the
|
||||
burst every item transcodes unthrottled, so the instantaneous-concurrency guarantee that
|
||||
`work_ahead_limit` provides is weaker than before — weaker, not absent, because `HlsSessionWorker`'s
|
||||
`transcodedBuffer <= 1min` gate still stops the loop at a 60 s buffer, leaving average CPU
|
||||
unchanged. Making the burst strictly cold-start-only would mean plumbing a "first process of this
|
||||
session" flag through `FFmpegState`; that complexity was not judged worth a bounded peak.
|
||||
- **Still images are excluded.** Their video input is paced by the realtime *filter* and takes no
|
||||
readrate at all, so a burst would only run the audio input ahead of the video for songs and offline
|
||||
filler, with no cold-start gain to show for it.
|
||||
- **Non-HLS realtime outputs (`TransportStream`, HLS-Direct) burst too**, since
|
||||
`FFmpegPlaybackSettingsCalculator` makes them unconditionally realtime. That is untested by the
|
||||
benchmark, which was segmenter-only; it is kept because the same first-read throttle delays those
|
||||
clients identically, and the outer `WrapSegmenter`/`Concat` processes still pace at `readrate 1.0`
|
||||
with no burst.
|
||||
- **Gated on runtime capability, not on a parsed version.** `-readrate_initial_burst` needs FFmpeg
|
||||
≥ 6.1, and `FFmpegKnownOption`/`HasOption` already existed for exactly this (its `AllOptions` list
|
||||
had simply been empty). Detection parses `ffmpeg -h long`, so an older binary silently keeps
|
||||
today's behavior instead of failing to start — the same fail-safe posture as the other capability
|
||||
gates, and cheaper to reason about than the version-string parsing in `NvidiaHardwareCapabilities`.
|
||||
|
||||
Reference in New Issue
Block a user