From 6c8c7feeba843ebf2a7887a59ddb524da6fa60f7 Mon Sep 17 00:00:00 2001 From: Timothy Date: Sat, 18 Jul 2026 15:30:56 +0200 Subject: [PATCH] feat(350): instrument HLS cold-start latency (phase split + feature flags) Adds one Information-level structured log per HLS tune-in cold-start so the real driver breakdown can be measured on prod before optimizing the transcode pipeline (measure-before-optimize). Log-only; no transcode behavior change. - WaitForPlaylistSegments returns a PlaylistSegmentsResult: Phase A (process startup -> playlist exists) vs Phase B (segment fill), segments reached, deadline-expired. - StartFFmpegSessionHandler emits one summary: total = setup + startup + fill, plus cleanly-detectable feature flags (subtitle burn-in, hwaccel family). - ColdStartFeatures: pure, unit-tested args->features helper (14 cases). Watermark / HDR->SDR / image-subtitle burn-in are deliberately not flagged (all reduce to overlay= in the args, indistinguishable); the full ffmpeg arguments remain available at Debug. Refs #350 (instrumentation slice; optimization deferred pending real data). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../Commands/StartFFmpegSessionHandler.cs | 36 +++++++- .../Streaming/HlsSessionWorker.cs | 27 +++++- .../FFmpeg/ColdStartFeaturesTests.cs | 89 +++++++++++++++++++ ErsatzTV.Core/FFmpeg/ColdStartFeatures.cs | 68 ++++++++++++++ .../FFmpeg/PlaylistSegmentsResult.cs | 16 ++++ .../Interfaces/FFmpeg/IHlsSessionWorker.cs | 4 +- 6 files changed, 233 insertions(+), 7 deletions(-) create mode 100644 ErsatzTV.Core.Tests/FFmpeg/ColdStartFeaturesTests.cs create mode 100644 ErsatzTV.Core/FFmpeg/ColdStartFeatures.cs create mode 100644 ErsatzTV.Core/FFmpeg/PlaylistSegmentsResult.cs diff --git a/ErsatzTV.Application/Streaming/Commands/StartFFmpegSessionHandler.cs b/ErsatzTV.Application/Streaming/Commands/StartFFmpegSessionHandler.cs index ecb944b0f..6cc07bb99 100644 --- a/ErsatzTV.Application/Streaming/Commands/StartFFmpegSessionHandler.cs +++ b/ErsatzTV.Application/Streaming/Commands/StartFFmpegSessionHandler.cs @@ -1,4 +1,5 @@ -using System.IO.Abstractions; +using System.Diagnostics; +using System.IO.Abstractions; using System.Threading.Channels; using ErsatzTV.Application.Channels; using ErsatzTV.Application.Graphics; @@ -75,6 +76,10 @@ public class StartFFmpegSessionHandler : IRequestHandler StartProcess(StartFFmpegSession request, CancellationToken cancellationToken) { + // measures the full client-visible cold-start: this handler only runs when the session + // is not already active, so its whole duration is the tune-in delay the client waits on + var coldStartStopwatch = Stopwatch.StartNew(); + Option idleTimeout = await _configElementRepository .GetValue(ConfigElementKey.FFmpegSegmenterTimeout, cancellationToken) .Map(maybeTimeout => maybeTimeout.Match(i => TimeSpan.FromSeconds(i), () => TimeSpan.FromMinutes(1))); @@ -114,7 +119,34 @@ public class StartFFmpegSessionHandler : IRequestHandler(ConfigElementKey.FFmpegInitialSegmentCount, cancellationToken) .Map(maybeCount => maybeCount.Match(identity, () => 1)); - await worker.WaitForPlaylistSegments(initialSegmentCount, cancellationToken); + PlaylistSegmentsResult segments = await worker.WaitForPlaylistSegments(initialSegmentCount, cancellationToken); + coldStartStopwatch.Stop(); + + // #350 cold-start instrumentation: one self-describing sample per tune-in so the real + // driver split (process startup vs segment fill, and which heavy features were active) + // can be measured on prod before any transcode-pipeline optimization. + // "setup" is the pre-wait handler overhead (config reads + framerate/channel/graphics + // mediator sends + worker spawn) so total = setup + startup + fill accounts for every ms. + long totalMs = (long)coldStartStopwatch.Elapsed.TotalMilliseconds; + long startupMs = (long)segments.ProcessStartup.TotalMilliseconds; + long fillMs = (long)segments.SegmentFill.TotalMilliseconds; + long setupMs = Math.Max(0, totalMs - startupMs - fillMs); + _logger.LogInformation( + "HLS cold-start channel {Channel} mode {Mode}: total {TotalMs}ms " + + "(setup {SetupMs}ms + startup {ProcessStartupMs}ms + fill {SegmentFillMs}ms), " + + "segments {SegmentsReached}/{InitialSegmentCount}, " + + "deadlineExpired {DeadlineExpired}, subtitleBurnIn {SubtitleBurnIn}, hwaccel {HwAccel}", + request.ChannelNumber, + request.Mode, + totalMs, + setupMs, + startupMs, + fillMs, + segments.SegmentsReached, + segments.InitialSegmentCount, + segments.DeadlineExpired, + segments.Features.SubtitleBurnIn, + segments.Features.HardwareAcceleration); return Unit.Default; } diff --git a/ErsatzTV.Application/Streaming/HlsSessionWorker.cs b/ErsatzTV.Application/Streaming/HlsSessionWorker.cs index 01bc4e741..8c2e68381 100644 --- a/ErsatzTV.Application/Streaming/HlsSessionWorker.cs +++ b/ErsatzTV.Application/Streaming/HlsSessionWorker.cs @@ -1,4 +1,4 @@ -using System.Diagnostics; +using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.IO.Abstractions; @@ -55,6 +55,12 @@ public class HlsSessionWorker : IHlsSessionWorker private string _workingDirectory; private Option _slugSeconds; + // The arguments of the first ffmpeg process launched for this session, captured for + // cold-start telemetry (#350). Written once on the sequential Run loop, read on the + // handler thread from WaitForPlaylistSegments after segments exist (a happens-before: + // segments cannot exist until this process ran) — volatile for cross-thread visibility. + private volatile string _coldStartFFmpegArguments; + public HlsSessionWorker( IServiceScopeFactory serviceScopeFactory, IGraphicsEngine graphicsEngine, @@ -300,17 +306,20 @@ public class HlsSessionWorker : IHlsSessionWorker } } - public async Task WaitForPlaylistSegments( + public async Task WaitForPlaylistSegments( int initialSegmentCount, CancellationToken cancellationToken) { _logger.LogDebug("Waiting for playlist segments..."); var sw = Stopwatch.StartNew(); + var processStartup = TimeSpan.Zero; + var segmentCount = 0; try { string playlistFileName = Path.Combine(_workingDirectory, "live.m3u8"); + // Phase A: ffmpeg process spawn -> playlist file exists (startup + probe + init + first GOP) _logger.LogDebug("Waiting for playlist to exist"); while (!_fileSystem.File.Exists(playlistFileName)) { @@ -318,12 +327,13 @@ public class HlsSessionWorker : IHlsSessionWorker } _logger.LogDebug("Playlist exists"); + processStartup = sw.Elapsed; // start the segment-wait deadline only after the playlist file appears, // so slow pipeline setup (e.g. h264 profile probing) doesn't consume the budget DateTimeOffset finish = DateTimeOffset.Now.AddSeconds(8); - var segmentCount = 0; + // Phase B: playlist exists -> the requested number of segments are present (or deadline) int lastSegmentCount = -1; while (DateTimeOffset.Now < finish && segmentCount < initialSegmentCount) { @@ -345,6 +355,14 @@ public class HlsSessionWorker : IHlsSessionWorker segmentCount = result.SegmentCount; } } + + return new PlaylistSegmentsResult( + processStartup, + sw.Elapsed - processStartup, + segmentCount, + initialSegmentCount, + segmentCount < initialSegmentCount, + ColdStartFeatures.FromFFmpegArguments(_coldStartFFmpegArguments)); } finally { @@ -533,6 +551,9 @@ public class HlsSessionWorker : IHlsSessionWorker Command process = processModel.Process; + // capture the first process's arguments once for cold-start telemetry (#350) + _coldStartFFmpegArguments ??= process.Arguments; + _logger.LogDebug("ffmpeg hls arguments {FFmpegArguments}", process.Arguments); try diff --git a/ErsatzTV.Core.Tests/FFmpeg/ColdStartFeaturesTests.cs b/ErsatzTV.Core.Tests/FFmpeg/ColdStartFeaturesTests.cs new file mode 100644 index 000000000..089365e10 --- /dev/null +++ b/ErsatzTV.Core.Tests/FFmpeg/ColdStartFeaturesTests.cs @@ -0,0 +1,89 @@ +using ErsatzTV.Core.FFmpeg; +using NUnit.Framework; +using Shouldly; + +namespace ErsatzTV.Core.Tests.FFmpeg; + +[TestFixture] +public class ColdStartFeaturesTests +{ + [Test] + public void Should_Detect_Subtitle_BurnIn_From_Subtitles_Filter() + { + const string Args = + "-i ep.mkv -filter_complex \"[0:0]subtitles=http://localhost:8409/media/subtitle/1.ass[v]\" " + + "-c:v h264_vaapi -f hls"; + + ColdStartFeatures features = ColdStartFeatures.FromFFmpegArguments(Args); + + features.SubtitleBurnIn.ShouldBeTrue(); + } + + [Test] + public void Should_Not_Flag_Subtitle_BurnIn_For_Copy_Or_Overlay() + { + // subtitle stream copy (-c:s) and a watermark/image-subtitle overlay must NOT be read as burn-in + const string Args = "-i ep.mkv -filter_complex \"[v][wm]overlay=10:10:format=0[vwm]\" -c:s copy -c:v libx264 -f hls"; + + ColdStartFeatures features = ColdStartFeatures.FromFFmpegArguments(Args); + + features.SubtitleBurnIn.ShouldBeFalse(); + } + + [TestCase("-c:v h264_vaapi -f hls", "vaapi")] + [TestCase("-c:v hevc_vaapi -f hls", "vaapi")] + [TestCase("-c:v h264_nvenc -f hls", "nvenc")] + [TestCase("-c:v h264_qsv -f hls", "qsv")] + [TestCase("-c:v h264_videotoolbox -f hls", "videotoolbox")] + [TestCase("-c:v h264_amf -f hls", "amf")] + public void Should_Detect_Hardware_Family_From_Encoder(string args, string expected) + { + ColdStartFeatures features = ColdStartFeatures.FromFFmpegArguments(args); + + features.HardwareAcceleration.ShouldBe(expected); + } + + [Test] + public void Should_Detect_Hardware_Family_From_Vaapi_Filter_Even_Without_Encoder_Token() + { + // a vaapi-accelerated pipeline surfaces the family via filters (e.g. scale_vaapi) too + const string Args = "-hwaccel vaapi -i ep.mkv -vf scale_vaapi=1920:1080 -f hls"; + + ColdStartFeatures features = ColdStartFeatures.FromFFmpegArguments(Args); + + features.HardwareAcceleration.ShouldBe("vaapi"); + } + + [Test] + public void Should_Fall_Back_To_Hwaccel_Flag_When_No_Hardware_Encoder_Or_Filter() + { + // hardware decode, software encode: attribute the decode accel + const string Args = "-hwaccel cuda -i ep.mkv -c:v libx264 -f hls"; + + ColdStartFeatures features = ColdStartFeatures.FromFFmpegArguments(Args); + + features.HardwareAcceleration.ShouldBe("cuda"); + } + + [Test] + public void Should_Report_Software_When_No_Acceleration() + { + const string Args = "-i ep.mkv -c:v libx264 -c:a aac -f hls"; + + ColdStartFeatures features = ColdStartFeatures.FromFFmpegArguments(Args); + + features.SubtitleBurnIn.ShouldBeFalse(); + features.HardwareAcceleration.ShouldBe("software"); + } + + [TestCase("")] + [TestCase(" ")] + [TestCase(null)] + public void Should_Report_Unknown_For_Empty_Arguments(string args) + { + ColdStartFeatures features = ColdStartFeatures.FromFFmpegArguments(args); + + features.SubtitleBurnIn.ShouldBeFalse(); + features.HardwareAcceleration.ShouldBe("unknown"); + } +} diff --git a/ErsatzTV.Core/FFmpeg/ColdStartFeatures.cs b/ErsatzTV.Core/FFmpeg/ColdStartFeatures.cs new file mode 100644 index 000000000..8169e6ce7 --- /dev/null +++ b/ErsatzTV.Core/FFmpeg/ColdStartFeatures.cs @@ -0,0 +1,68 @@ +namespace ErsatzTV.Core.FFmpeg; + +/// +/// Heavy transcode features detected from a cold-start's FFmpeg arguments, so each HLS +/// cold-start latency sample is self-describing (#350). Only cleanly-detectable signals are +/// surfaced: subtitle burn-in (the libass subtitles= text filter) and the hardware +/// acceleration family. Watermark, HDR->SDR, and image-based subtitle burn-in are +/// intentionally NOT flagged here: in the argument string they all reduce to overlay=, +/// indistinguishable from one another, so a boolean would mislead the very measurement it +/// serves. Inspect the full ffmpeg hls arguments debug log for those details. +/// Detection is a case-sensitive substring scan of the whole argument string (which includes +/// the input media path), so a media filename literally containing e.g. subtitles= or +/// _amf could false-positive; this is telemetry-only (no behavior impact) and low-probability. +/// +public readonly record struct ColdStartFeatures(bool SubtitleBurnIn, string HardwareAcceleration) +{ + // Encoder/filter suffix -> accel family. Matching any of these substrings means the + // pipeline is that hardware family (the encoder codec, e.g. h264_vaapi, dominates + // cold-start encoder init cost; VAAPI/QSV/CUDA filters carry the same suffix). + private static readonly (string Token, string Family)[] HardwareFamilies = + [ + ("_vaapi", "vaapi"), + ("_nvenc", "nvenc"), + ("_qsv", "qsv"), + ("_videotoolbox", "videotoolbox"), + ("_amf", "amf"), + ("_rkmpp", "rkmpp") + ]; + + public static ColdStartFeatures FromFFmpegArguments(string arguments) + { + if (string.IsNullOrWhiteSpace(arguments)) + { + return new ColdStartFeatures(false, "unknown"); + } + + bool subtitleBurnIn = arguments.Contains("subtitles=", StringComparison.Ordinal); + return new ColdStartFeatures(subtitleBurnIn, DetectHardwareAcceleration(arguments)); + } + + private static string DetectHardwareAcceleration(string arguments) + { + foreach ((string token, string family) in HardwareFamilies) + { + if (arguments.Contains(token, StringComparison.Ordinal)) + { + return family; + } + } + + // No hardware encoder/filter present: fall back to the decode accel (a hardware + // decode + software encode still pays an init cost worth attributing). + const string HwAccelFlag = "-hwaccel "; + int index = arguments.IndexOf(HwAccelFlag, StringComparison.Ordinal); + if (index >= 0) + { + string rest = arguments[(index + HwAccelFlag.Length)..].TrimStart(); + int end = rest.IndexOf(' '); + string value = end >= 0 ? rest[..end] : rest; + if (value.Length > 0) + { + return value; + } + } + + return "software"; + } +} diff --git a/ErsatzTV.Core/FFmpeg/PlaylistSegmentsResult.cs b/ErsatzTV.Core/FFmpeg/PlaylistSegmentsResult.cs new file mode 100644 index 000000000..83964430a --- /dev/null +++ b/ErsatzTV.Core/FFmpeg/PlaylistSegmentsResult.cs @@ -0,0 +1,16 @@ +namespace ErsatzTV.Core.FFmpeg; + +/// +/// Timing + descriptive result of an HLS cold-start's wait-for-first-segments phase (#350). +/// is Phase A (wait entry -> playlist file exists — approximately +/// FFmpeg process spawn + probe + libass/encoder init + first GOP, since the wait begins right +/// after the fire-and-forget worker is launched); is Phase B (playlist +/// exists -> the requested number of segments are present, or the 8s deadline). +/// +public readonly record struct PlaylistSegmentsResult( + TimeSpan ProcessStartup, + TimeSpan SegmentFill, + int SegmentsReached, + int InitialSegmentCount, + bool DeadlineExpired, + ColdStartFeatures Features); diff --git a/ErsatzTV.Core/Interfaces/FFmpeg/IHlsSessionWorker.cs b/ErsatzTV.Core/Interfaces/FFmpeg/IHlsSessionWorker.cs index a67c91db8..d6879ed1d 100644 --- a/ErsatzTV.Core/Interfaces/FFmpeg/IHlsSessionWorker.cs +++ b/ErsatzTV.Core/Interfaces/FFmpeg/IHlsSessionWorker.cs @@ -1,4 +1,4 @@ -using ErsatzTV.Core.FFmpeg; +using ErsatzTV.Core.FFmpeg; namespace ErsatzTV.Core.Interfaces.FFmpeg; @@ -10,5 +10,5 @@ public interface IHlsSessionWorker : IDisposable void PlayoutUpdated(); HlsSessionModel GetModel(); Task Run(string channelNumber, Option idleTimeout, CancellationToken incomingCancellationToken); - Task WaitForPlaylistSegments(int initialSegmentCount, CancellationToken cancellationToken); + Task WaitForPlaylistSegments(int initialSegmentCount, CancellationToken cancellationToken); } -- 2.47.3