feat(350): instrument HLS cold-start latency (phase split + feature flags) #446

Merged
timothy merged 1 commits from feat/350-coldstart-instrumentation into main 2026-07-18 16:16:39 +02:00
6 changed files with 233 additions and 7 deletions
@@ -1,4 +1,5 @@
using System.IO.Abstractions;
using System.Diagnostics;
using System.IO.Abstractions;
using System.Threading.Channels;
using ErsatzTV.Application.Channels;
using ErsatzTV.Application.Graphics;
@@ -75,6 +76,10 @@ public class StartFFmpegSessionHandler : IRequestHandler<StartFFmpegSession, Eit
private async Task<Unit> StartProcess(StartFFmpegSession request, CancellationToken cancellationToken)
{
// measures the full client-visible cold-start: this handler only runs when the session
// is not already active, so its whole duration is the tune-in delay the client waits on
var coldStartStopwatch = Stopwatch.StartNew();
Option<TimeSpan> idleTimeout = await _configElementRepository
.GetValue<int>(ConfigElementKey.FFmpegSegmenterTimeout, cancellationToken)
.Map(maybeTimeout => maybeTimeout.Match(i => TimeSpan.FromSeconds(i), () => TimeSpan.FromMinutes(1)));
@@ -114,7 +119,34 @@ public class StartFFmpegSessionHandler : IRequestHandler<StartFFmpegSession, Eit
.GetValue<int>(ConfigElementKey.FFmpegInitialSegmentCount, cancellationToken)
.Map(maybeCount => maybeCount.Match(identity, () => 1));
await worker.WaitForPlaylistSegments(initialSegmentCount, cancellationToken);
PlaylistSegmentsResult segments = await worker.WaitForPlaylistSegments(initialSegmentCount, cancellationToken);
coldStartStopwatch.Stop();
// #350 cold-start instrumentation: one self-describing sample per tune-in so the real
// driver split (process startup vs segment fill, and which heavy features were active)
// can be measured on prod before any transcode-pipeline optimization.
// "setup" is the pre-wait handler overhead (config reads + framerate/channel/graphics
// mediator sends + worker spawn) so total = setup + startup + fill accounts for every ms.
long totalMs = (long)coldStartStopwatch.Elapsed.TotalMilliseconds;
long startupMs = (long)segments.ProcessStartup.TotalMilliseconds;
long fillMs = (long)segments.SegmentFill.TotalMilliseconds;
long setupMs = Math.Max(0, totalMs - startupMs - fillMs);
_logger.LogInformation(
"HLS cold-start channel {Channel} mode {Mode}: total {TotalMs}ms " +
"(setup {SetupMs}ms + startup {ProcessStartupMs}ms + fill {SegmentFillMs}ms), " +
"segments {SegmentsReached}/{InitialSegmentCount}, " +
"deadlineExpired {DeadlineExpired}, subtitleBurnIn {SubtitleBurnIn}, hwaccel {HwAccel}",
request.ChannelNumber,
request.Mode,
totalMs,
setupMs,
startupMs,
fillMs,
segments.SegmentsReached,
segments.InitialSegmentCount,
segments.DeadlineExpired,
segments.Features.SubtitleBurnIn,
segments.Features.HardwareAcceleration);
return Unit.Default;
}
@@ -1,4 +1,4 @@
using System.Diagnostics;
using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
using System.Globalization;
using System.IO.Abstractions;
@@ -55,6 +55,12 @@ public class HlsSessionWorker : IHlsSessionWorker
private string _workingDirectory;
private Option<double> _slugSeconds;
// The arguments of the first ffmpeg process launched for this session, captured for
// cold-start telemetry (#350). Written once on the sequential Run loop, read on the
// handler thread from WaitForPlaylistSegments after segments exist (a happens-before:
// segments cannot exist until this process ran) — volatile for cross-thread visibility.
private volatile string _coldStartFFmpegArguments;
public HlsSessionWorker(
IServiceScopeFactory serviceScopeFactory,
IGraphicsEngine graphicsEngine,
@@ -300,17 +306,20 @@ public class HlsSessionWorker : IHlsSessionWorker
}
}
public async Task WaitForPlaylistSegments(
public async Task<PlaylistSegmentsResult> WaitForPlaylistSegments(
int initialSegmentCount,
CancellationToken cancellationToken)
{
_logger.LogDebug("Waiting for playlist segments...");
var sw = Stopwatch.StartNew();
var processStartup = TimeSpan.Zero;
var segmentCount = 0;
try
{
string playlistFileName = Path.Combine(_workingDirectory, "live.m3u8");
// Phase A: ffmpeg process spawn -> playlist file exists (startup + probe + init + first GOP)
_logger.LogDebug("Waiting for playlist to exist");
while (!_fileSystem.File.Exists(playlistFileName))
{
@@ -318,12 +327,13 @@ public class HlsSessionWorker : IHlsSessionWorker
}
_logger.LogDebug("Playlist exists");
processStartup = sw.Elapsed;
// start the segment-wait deadline only after the playlist file appears,
// so slow pipeline setup (e.g. h264 profile probing) doesn't consume the budget
DateTimeOffset finish = DateTimeOffset.Now.AddSeconds(8);
var segmentCount = 0;
// Phase B: playlist exists -> the requested number of segments are present (or deadline)
int lastSegmentCount = -1;
while (DateTimeOffset.Now < finish && segmentCount < initialSegmentCount)
{
@@ -345,6 +355,14 @@ public class HlsSessionWorker : IHlsSessionWorker
segmentCount = result.SegmentCount;
}
}
return new PlaylistSegmentsResult(
processStartup,
sw.Elapsed - processStartup,
segmentCount,
initialSegmentCount,
segmentCount < initialSegmentCount,
ColdStartFeatures.FromFFmpegArguments(_coldStartFFmpegArguments));
}
finally
{
@@ -533,6 +551,9 @@ public class HlsSessionWorker : IHlsSessionWorker
Command process = processModel.Process;
// capture the first process's arguments once for cold-start telemetry (#350)
_coldStartFFmpegArguments ??= process.Arguments;
_logger.LogDebug("ffmpeg hls arguments {FFmpegArguments}", process.Arguments);
try
@@ -0,0 +1,89 @@
using ErsatzTV.Core.FFmpeg;
using NUnit.Framework;
using Shouldly;
namespace ErsatzTV.Core.Tests.FFmpeg;
[TestFixture]
public class ColdStartFeaturesTests
{
[Test]
public void Should_Detect_Subtitle_BurnIn_From_Subtitles_Filter()
{
const string Args =
"-i ep.mkv -filter_complex \"[0:0]subtitles=http://localhost:8409/media/subtitle/1.ass[v]\" " +
"-c:v h264_vaapi -f hls";
ColdStartFeatures features = ColdStartFeatures.FromFFmpegArguments(Args);
features.SubtitleBurnIn.ShouldBeTrue();
}
[Test]
public void Should_Not_Flag_Subtitle_BurnIn_For_Copy_Or_Overlay()
{
// subtitle stream copy (-c:s) and a watermark/image-subtitle overlay must NOT be read as burn-in
const string Args = "-i ep.mkv -filter_complex \"[v][wm]overlay=10:10:format=0[vwm]\" -c:s copy -c:v libx264 -f hls";
ColdStartFeatures features = ColdStartFeatures.FromFFmpegArguments(Args);
features.SubtitleBurnIn.ShouldBeFalse();
}
[TestCase("-c:v h264_vaapi -f hls", "vaapi")]
[TestCase("-c:v hevc_vaapi -f hls", "vaapi")]
[TestCase("-c:v h264_nvenc -f hls", "nvenc")]
[TestCase("-c:v h264_qsv -f hls", "qsv")]
[TestCase("-c:v h264_videotoolbox -f hls", "videotoolbox")]
[TestCase("-c:v h264_amf -f hls", "amf")]
public void Should_Detect_Hardware_Family_From_Encoder(string args, string expected)
{
ColdStartFeatures features = ColdStartFeatures.FromFFmpegArguments(args);
features.HardwareAcceleration.ShouldBe(expected);
}
[Test]
public void Should_Detect_Hardware_Family_From_Vaapi_Filter_Even_Without_Encoder_Token()
{
// a vaapi-accelerated pipeline surfaces the family via filters (e.g. scale_vaapi) too
const string Args = "-hwaccel vaapi -i ep.mkv -vf scale_vaapi=1920:1080 -f hls";
ColdStartFeatures features = ColdStartFeatures.FromFFmpegArguments(Args);
features.HardwareAcceleration.ShouldBe("vaapi");
}
[Test]
public void Should_Fall_Back_To_Hwaccel_Flag_When_No_Hardware_Encoder_Or_Filter()
{
// hardware decode, software encode: attribute the decode accel
const string Args = "-hwaccel cuda -i ep.mkv -c:v libx264 -f hls";
ColdStartFeatures features = ColdStartFeatures.FromFFmpegArguments(Args);
features.HardwareAcceleration.ShouldBe("cuda");
}
[Test]
public void Should_Report_Software_When_No_Acceleration()
{
const string Args = "-i ep.mkv -c:v libx264 -c:a aac -f hls";
ColdStartFeatures features = ColdStartFeatures.FromFFmpegArguments(Args);
features.SubtitleBurnIn.ShouldBeFalse();
features.HardwareAcceleration.ShouldBe("software");
}
[TestCase("")]
[TestCase(" ")]
[TestCase(null)]
public void Should_Report_Unknown_For_Empty_Arguments(string args)
{
ColdStartFeatures features = ColdStartFeatures.FromFFmpegArguments(args);
features.SubtitleBurnIn.ShouldBeFalse();
features.HardwareAcceleration.ShouldBe("unknown");
}
}
+68
View File
@@ -0,0 +1,68 @@
namespace ErsatzTV.Core.FFmpeg;
/// <summary>
/// Heavy transcode features detected from a cold-start's FFmpeg arguments, so each HLS
/// cold-start latency sample is self-describing (#350). Only cleanly-detectable signals are
/// surfaced: subtitle burn-in (the libass <c>subtitles=</c> text filter) and the hardware
/// acceleration family. Watermark, HDR-&gt;SDR, and <em>image</em>-based subtitle burn-in are
/// intentionally NOT flagged here: in the argument string they all reduce to <c>overlay=</c>,
/// indistinguishable from one another, so a boolean would mislead the very measurement it
/// serves. Inspect the full <c>ffmpeg hls arguments</c> debug log for those details.
/// Detection is a case-sensitive substring scan of the whole argument string (which includes
/// the input media path), so a media filename literally containing e.g. <c>subtitles=</c> or
/// <c>_amf</c> could false-positive; this is telemetry-only (no behavior impact) and low-probability.
/// </summary>
public readonly record struct ColdStartFeatures(bool SubtitleBurnIn, string HardwareAcceleration)
{
// Encoder/filter suffix -> accel family. Matching any of these substrings means the
// pipeline is that hardware family (the encoder codec, e.g. h264_vaapi, dominates
// cold-start encoder init cost; VAAPI/QSV/CUDA filters carry the same suffix).
private static readonly (string Token, string Family)[] HardwareFamilies =
[
("_vaapi", "vaapi"),
("_nvenc", "nvenc"),
("_qsv", "qsv"),
("_videotoolbox", "videotoolbox"),
("_amf", "amf"),
("_rkmpp", "rkmpp")
];
public static ColdStartFeatures FromFFmpegArguments(string arguments)
{
if (string.IsNullOrWhiteSpace(arguments))
{
return new ColdStartFeatures(false, "unknown");
}
bool subtitleBurnIn = arguments.Contains("subtitles=", StringComparison.Ordinal);
return new ColdStartFeatures(subtitleBurnIn, DetectHardwareAcceleration(arguments));
}
private static string DetectHardwareAcceleration(string arguments)
{
foreach ((string token, string family) in HardwareFamilies)
{
if (arguments.Contains(token, StringComparison.Ordinal))
{
return family;
}
}
// No hardware encoder/filter present: fall back to the decode accel (a hardware
// decode + software encode still pays an init cost worth attributing).
const string HwAccelFlag = "-hwaccel ";
int index = arguments.IndexOf(HwAccelFlag, StringComparison.Ordinal);
if (index >= 0)
{
string rest = arguments[(index + HwAccelFlag.Length)..].TrimStart();
int end = rest.IndexOf(' ');
string value = end >= 0 ? rest[..end] : rest;
if (value.Length > 0)
{
return value;
}
}
return "software";
}
}
@@ -0,0 +1,16 @@
namespace ErsatzTV.Core.FFmpeg;
/// <summary>
/// Timing + descriptive result of an HLS cold-start's wait-for-first-segments phase (#350).
/// <see cref="ProcessStartup"/> is Phase A (wait entry -&gt; playlist file exists — approximately
/// FFmpeg process spawn + probe + libass/encoder init + first GOP, since the wait begins right
/// after the fire-and-forget worker is launched); <see cref="SegmentFill"/> is Phase B (playlist
/// exists -&gt; the requested number of segments are present, or the 8s deadline).
/// </summary>
public readonly record struct PlaylistSegmentsResult(
TimeSpan ProcessStartup,
TimeSpan SegmentFill,
int SegmentsReached,
int InitialSegmentCount,
bool DeadlineExpired,
ColdStartFeatures Features);
@@ -1,4 +1,4 @@
using ErsatzTV.Core.FFmpeg;
using ErsatzTV.Core.FFmpeg;
namespace ErsatzTV.Core.Interfaces.FFmpeg;
@@ -10,5 +10,5 @@ public interface IHlsSessionWorker : IDisposable
void PlayoutUpdated();
HlsSessionModel GetModel();
Task Run(string channelNumber, Option<TimeSpan> idleTimeout, CancellationToken incomingCancellationToken);
Task WaitForPlaylistSegments(int initialSegmentCount, CancellationToken cancellationToken);
Task<PlaylistSegmentsResult> WaitForPlaylistSegments(int initialSegmentCount, CancellationToken cancellationToken);
}