feat(350): instrument HLS cold-start latency (phase split + feature flags) #446
@@ -1,4 +1,5 @@
|
||||
using System.IO.Abstractions;
|
||||
using System.Diagnostics;
|
||||
using System.IO.Abstractions;
|
||||
using System.Threading.Channels;
|
||||
using ErsatzTV.Application.Channels;
|
||||
using ErsatzTV.Application.Graphics;
|
||||
@@ -75,6 +76,10 @@ public class StartFFmpegSessionHandler : IRequestHandler<StartFFmpegSession, Eit
|
||||
|
||||
private async Task<Unit> StartProcess(StartFFmpegSession request, CancellationToken cancellationToken)
|
||||
{
|
||||
// measures the full client-visible cold-start: this handler only runs when the session
|
||||
// is not already active, so its whole duration is the tune-in delay the client waits on
|
||||
var coldStartStopwatch = Stopwatch.StartNew();
|
||||
|
||||
Option<TimeSpan> idleTimeout = await _configElementRepository
|
||||
.GetValue<int>(ConfigElementKey.FFmpegSegmenterTimeout, cancellationToken)
|
||||
.Map(maybeTimeout => maybeTimeout.Match(i => TimeSpan.FromSeconds(i), () => TimeSpan.FromMinutes(1)));
|
||||
@@ -114,7 +119,34 @@ public class StartFFmpegSessionHandler : IRequestHandler<StartFFmpegSession, Eit
|
||||
.GetValue<int>(ConfigElementKey.FFmpegInitialSegmentCount, cancellationToken)
|
||||
.Map(maybeCount => maybeCount.Match(identity, () => 1));
|
||||
|
||||
await worker.WaitForPlaylistSegments(initialSegmentCount, cancellationToken);
|
||||
PlaylistSegmentsResult segments = await worker.WaitForPlaylistSegments(initialSegmentCount, cancellationToken);
|
||||
coldStartStopwatch.Stop();
|
||||
|
||||
// #350 cold-start instrumentation: one self-describing sample per tune-in so the real
|
||||
// driver split (process startup vs segment fill, and which heavy features were active)
|
||||
// can be measured on prod before any transcode-pipeline optimization.
|
||||
// "setup" is the pre-wait handler overhead (config reads + framerate/channel/graphics
|
||||
// mediator sends + worker spawn) so total = setup + startup + fill accounts for every ms.
|
||||
long totalMs = (long)coldStartStopwatch.Elapsed.TotalMilliseconds;
|
||||
long startupMs = (long)segments.ProcessStartup.TotalMilliseconds;
|
||||
long fillMs = (long)segments.SegmentFill.TotalMilliseconds;
|
||||
long setupMs = Math.Max(0, totalMs - startupMs - fillMs);
|
||||
_logger.LogInformation(
|
||||
"HLS cold-start channel {Channel} mode {Mode}: total {TotalMs}ms " +
|
||||
"(setup {SetupMs}ms + startup {ProcessStartupMs}ms + fill {SegmentFillMs}ms), " +
|
||||
"segments {SegmentsReached}/{InitialSegmentCount}, " +
|
||||
"deadlineExpired {DeadlineExpired}, subtitleBurnIn {SubtitleBurnIn}, hwaccel {HwAccel}",
|
||||
request.ChannelNumber,
|
||||
request.Mode,
|
||||
totalMs,
|
||||
setupMs,
|
||||
startupMs,
|
||||
fillMs,
|
||||
segments.SegmentsReached,
|
||||
segments.InitialSegmentCount,
|
||||
segments.DeadlineExpired,
|
||||
segments.Features.SubtitleBurnIn,
|
||||
segments.Features.HardwareAcceleration);
|
||||
|
||||
return Unit.Default;
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
using System.Diagnostics;
|
||||
using System.Diagnostics;
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
using System.Globalization;
|
||||
using System.IO.Abstractions;
|
||||
@@ -55,6 +55,12 @@ public class HlsSessionWorker : IHlsSessionWorker
|
||||
private string _workingDirectory;
|
||||
private Option<double> _slugSeconds;
|
||||
|
||||
// The arguments of the first ffmpeg process launched for this session, captured for
|
||||
// cold-start telemetry (#350). Written once on the sequential Run loop, read on the
|
||||
// handler thread from WaitForPlaylistSegments after segments exist (a happens-before:
|
||||
// segments cannot exist until this process ran) — volatile for cross-thread visibility.
|
||||
private volatile string _coldStartFFmpegArguments;
|
||||
|
||||
public HlsSessionWorker(
|
||||
IServiceScopeFactory serviceScopeFactory,
|
||||
IGraphicsEngine graphicsEngine,
|
||||
@@ -300,17 +306,20 @@ public class HlsSessionWorker : IHlsSessionWorker
|
||||
}
|
||||
}
|
||||
|
||||
public async Task WaitForPlaylistSegments(
|
||||
public async Task<PlaylistSegmentsResult> WaitForPlaylistSegments(
|
||||
int initialSegmentCount,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
_logger.LogDebug("Waiting for playlist segments...");
|
||||
|
||||
var sw = Stopwatch.StartNew();
|
||||
var processStartup = TimeSpan.Zero;
|
||||
var segmentCount = 0;
|
||||
try
|
||||
{
|
||||
string playlistFileName = Path.Combine(_workingDirectory, "live.m3u8");
|
||||
|
||||
// Phase A: ffmpeg process spawn -> playlist file exists (startup + probe + init + first GOP)
|
||||
_logger.LogDebug("Waiting for playlist to exist");
|
||||
while (!_fileSystem.File.Exists(playlistFileName))
|
||||
{
|
||||
@@ -318,12 +327,13 @@ public class HlsSessionWorker : IHlsSessionWorker
|
||||
}
|
||||
|
||||
_logger.LogDebug("Playlist exists");
|
||||
processStartup = sw.Elapsed;
|
||||
|
||||
// start the segment-wait deadline only after the playlist file appears,
|
||||
// so slow pipeline setup (e.g. h264 profile probing) doesn't consume the budget
|
||||
DateTimeOffset finish = DateTimeOffset.Now.AddSeconds(8);
|
||||
|
||||
var segmentCount = 0;
|
||||
// Phase B: playlist exists -> the requested number of segments are present (or deadline)
|
||||
int lastSegmentCount = -1;
|
||||
while (DateTimeOffset.Now < finish && segmentCount < initialSegmentCount)
|
||||
{
|
||||
@@ -345,6 +355,14 @@ public class HlsSessionWorker : IHlsSessionWorker
|
||||
segmentCount = result.SegmentCount;
|
||||
}
|
||||
}
|
||||
|
||||
return new PlaylistSegmentsResult(
|
||||
processStartup,
|
||||
sw.Elapsed - processStartup,
|
||||
segmentCount,
|
||||
initialSegmentCount,
|
||||
segmentCount < initialSegmentCount,
|
||||
ColdStartFeatures.FromFFmpegArguments(_coldStartFFmpegArguments));
|
||||
}
|
||||
finally
|
||||
{
|
||||
@@ -533,6 +551,9 @@ public class HlsSessionWorker : IHlsSessionWorker
|
||||
|
||||
Command process = processModel.Process;
|
||||
|
||||
// capture the first process's arguments once for cold-start telemetry (#350)
|
||||
_coldStartFFmpegArguments ??= process.Arguments;
|
||||
|
||||
_logger.LogDebug("ffmpeg hls arguments {FFmpegArguments}", process.Arguments);
|
||||
|
||||
try
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
using ErsatzTV.Core.FFmpeg;
|
||||
using NUnit.Framework;
|
||||
using Shouldly;
|
||||
|
||||
namespace ErsatzTV.Core.Tests.FFmpeg;
|
||||
|
||||
[TestFixture]
|
||||
public class ColdStartFeaturesTests
|
||||
{
|
||||
[Test]
|
||||
public void Should_Detect_Subtitle_BurnIn_From_Subtitles_Filter()
|
||||
{
|
||||
const string Args =
|
||||
"-i ep.mkv -filter_complex \"[0:0]subtitles=http://localhost:8409/media/subtitle/1.ass[v]\" " +
|
||||
"-c:v h264_vaapi -f hls";
|
||||
|
||||
ColdStartFeatures features = ColdStartFeatures.FromFFmpegArguments(Args);
|
||||
|
||||
features.SubtitleBurnIn.ShouldBeTrue();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void Should_Not_Flag_Subtitle_BurnIn_For_Copy_Or_Overlay()
|
||||
{
|
||||
// subtitle stream copy (-c:s) and a watermark/image-subtitle overlay must NOT be read as burn-in
|
||||
const string Args = "-i ep.mkv -filter_complex \"[v][wm]overlay=10:10:format=0[vwm]\" -c:s copy -c:v libx264 -f hls";
|
||||
|
||||
ColdStartFeatures features = ColdStartFeatures.FromFFmpegArguments(Args);
|
||||
|
||||
features.SubtitleBurnIn.ShouldBeFalse();
|
||||
}
|
||||
|
||||
[TestCase("-c:v h264_vaapi -f hls", "vaapi")]
|
||||
[TestCase("-c:v hevc_vaapi -f hls", "vaapi")]
|
||||
[TestCase("-c:v h264_nvenc -f hls", "nvenc")]
|
||||
[TestCase("-c:v h264_qsv -f hls", "qsv")]
|
||||
[TestCase("-c:v h264_videotoolbox -f hls", "videotoolbox")]
|
||||
[TestCase("-c:v h264_amf -f hls", "amf")]
|
||||
public void Should_Detect_Hardware_Family_From_Encoder(string args, string expected)
|
||||
{
|
||||
ColdStartFeatures features = ColdStartFeatures.FromFFmpegArguments(args);
|
||||
|
||||
features.HardwareAcceleration.ShouldBe(expected);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void Should_Detect_Hardware_Family_From_Vaapi_Filter_Even_Without_Encoder_Token()
|
||||
{
|
||||
// a vaapi-accelerated pipeline surfaces the family via filters (e.g. scale_vaapi) too
|
||||
const string Args = "-hwaccel vaapi -i ep.mkv -vf scale_vaapi=1920:1080 -f hls";
|
||||
|
||||
ColdStartFeatures features = ColdStartFeatures.FromFFmpegArguments(Args);
|
||||
|
||||
features.HardwareAcceleration.ShouldBe("vaapi");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void Should_Fall_Back_To_Hwaccel_Flag_When_No_Hardware_Encoder_Or_Filter()
|
||||
{
|
||||
// hardware decode, software encode: attribute the decode accel
|
||||
const string Args = "-hwaccel cuda -i ep.mkv -c:v libx264 -f hls";
|
||||
|
||||
ColdStartFeatures features = ColdStartFeatures.FromFFmpegArguments(Args);
|
||||
|
||||
features.HardwareAcceleration.ShouldBe("cuda");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void Should_Report_Software_When_No_Acceleration()
|
||||
{
|
||||
const string Args = "-i ep.mkv -c:v libx264 -c:a aac -f hls";
|
||||
|
||||
ColdStartFeatures features = ColdStartFeatures.FromFFmpegArguments(Args);
|
||||
|
||||
features.SubtitleBurnIn.ShouldBeFalse();
|
||||
features.HardwareAcceleration.ShouldBe("software");
|
||||
}
|
||||
|
||||
[TestCase("")]
|
||||
[TestCase(" ")]
|
||||
[TestCase(null)]
|
||||
public void Should_Report_Unknown_For_Empty_Arguments(string args)
|
||||
{
|
||||
ColdStartFeatures features = ColdStartFeatures.FromFFmpegArguments(args);
|
||||
|
||||
features.SubtitleBurnIn.ShouldBeFalse();
|
||||
features.HardwareAcceleration.ShouldBe("unknown");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
namespace ErsatzTV.Core.FFmpeg;
|
||||
|
||||
/// <summary>
|
||||
/// Heavy transcode features detected from a cold-start's FFmpeg arguments, so each HLS
|
||||
/// cold-start latency sample is self-describing (#350). Only cleanly-detectable signals are
|
||||
/// surfaced: subtitle burn-in (the libass <c>subtitles=</c> text filter) and the hardware
|
||||
/// acceleration family. Watermark, HDR->SDR, and <em>image</em>-based subtitle burn-in are
|
||||
/// intentionally NOT flagged here: in the argument string they all reduce to <c>overlay=</c>,
|
||||
/// indistinguishable from one another, so a boolean would mislead the very measurement it
|
||||
/// serves. Inspect the full <c>ffmpeg hls arguments</c> debug log for those details.
|
||||
/// Detection is a case-sensitive substring scan of the whole argument string (which includes
|
||||
/// the input media path), so a media filename literally containing e.g. <c>subtitles=</c> or
|
||||
/// <c>_amf</c> could false-positive; this is telemetry-only (no behavior impact) and low-probability.
|
||||
/// </summary>
|
||||
public readonly record struct ColdStartFeatures(bool SubtitleBurnIn, string HardwareAcceleration)
|
||||
{
|
||||
// Encoder/filter suffix -> accel family. Matching any of these substrings means the
|
||||
// pipeline is that hardware family (the encoder codec, e.g. h264_vaapi, dominates
|
||||
// cold-start encoder init cost; VAAPI/QSV/CUDA filters carry the same suffix).
|
||||
private static readonly (string Token, string Family)[] HardwareFamilies =
|
||||
[
|
||||
("_vaapi", "vaapi"),
|
||||
("_nvenc", "nvenc"),
|
||||
("_qsv", "qsv"),
|
||||
("_videotoolbox", "videotoolbox"),
|
||||
("_amf", "amf"),
|
||||
("_rkmpp", "rkmpp")
|
||||
];
|
||||
|
||||
public static ColdStartFeatures FromFFmpegArguments(string arguments)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(arguments))
|
||||
{
|
||||
return new ColdStartFeatures(false, "unknown");
|
||||
}
|
||||
|
||||
bool subtitleBurnIn = arguments.Contains("subtitles=", StringComparison.Ordinal);
|
||||
return new ColdStartFeatures(subtitleBurnIn, DetectHardwareAcceleration(arguments));
|
||||
}
|
||||
|
||||
private static string DetectHardwareAcceleration(string arguments)
|
||||
{
|
||||
foreach ((string token, string family) in HardwareFamilies)
|
||||
{
|
||||
if (arguments.Contains(token, StringComparison.Ordinal))
|
||||
{
|
||||
return family;
|
||||
}
|
||||
}
|
||||
|
||||
// No hardware encoder/filter present: fall back to the decode accel (a hardware
|
||||
// decode + software encode still pays an init cost worth attributing).
|
||||
const string HwAccelFlag = "-hwaccel ";
|
||||
int index = arguments.IndexOf(HwAccelFlag, StringComparison.Ordinal);
|
||||
if (index >= 0)
|
||||
{
|
||||
string rest = arguments[(index + HwAccelFlag.Length)..].TrimStart();
|
||||
int end = rest.IndexOf(' ');
|
||||
string value = end >= 0 ? rest[..end] : rest;
|
||||
if (value.Length > 0)
|
||||
{
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
return "software";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
namespace ErsatzTV.Core.FFmpeg;
|
||||
|
||||
/// <summary>
|
||||
/// Timing + descriptive result of an HLS cold-start's wait-for-first-segments phase (#350).
|
||||
/// <see cref="ProcessStartup"/> is Phase A (wait entry -> playlist file exists — approximately
|
||||
/// FFmpeg process spawn + probe + libass/encoder init + first GOP, since the wait begins right
|
||||
/// after the fire-and-forget worker is launched); <see cref="SegmentFill"/> is Phase B (playlist
|
||||
/// exists -> the requested number of segments are present, or the 8s deadline).
|
||||
/// </summary>
|
||||
public readonly record struct PlaylistSegmentsResult(
|
||||
TimeSpan ProcessStartup,
|
||||
TimeSpan SegmentFill,
|
||||
int SegmentsReached,
|
||||
int InitialSegmentCount,
|
||||
bool DeadlineExpired,
|
||||
ColdStartFeatures Features);
|
||||
@@ -1,4 +1,4 @@
|
||||
using ErsatzTV.Core.FFmpeg;
|
||||
using ErsatzTV.Core.FFmpeg;
|
||||
|
||||
namespace ErsatzTV.Core.Interfaces.FFmpeg;
|
||||
|
||||
@@ -10,5 +10,5 @@ public interface IHlsSessionWorker : IDisposable
|
||||
void PlayoutUpdated();
|
||||
HlsSessionModel GetModel();
|
||||
Task Run(string channelNumber, Option<TimeSpan> idleTimeout, CancellationToken incomingCancellationToken);
|
||||
Task WaitForPlaylistSegments(int initialSegmentCount, CancellationToken cancellationToken);
|
||||
Task<PlaylistSegmentsResult> WaitForPlaylistSegments(int initialSegmentCount, CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user