From bc225d35facdced89e0a25e2515dff9fe436e372 Mon Sep 17 00:00:00 2001 From: Jason Dove Date: Tue, 8 Mar 2022 12:52:29 -0600 Subject: [PATCH] add troubleshooting code to hls segmenter (#685) --- .../Streaming/HlsSessionWorker.cs | 26 +-- .../Streaming/Queries/GetLastPtsDuration.cs | 2 +- .../Queries/GetLastPtsDurationHandler.cs | 169 +++++++++++++----- ErsatzTV.Core/FFmpeg/TempFileCategory.cs | 1 + 4 files changed, 141 insertions(+), 57 deletions(-) diff --git a/ErsatzTV.Application/Streaming/HlsSessionWorker.cs b/ErsatzTV.Application/Streaming/HlsSessionWorker.cs index 6619b226f..7060396fb 100644 --- a/ErsatzTV.Application/Streaming/HlsSessionWorker.cs +++ b/ErsatzTV.Application/Streaming/HlsSessionWorker.cs @@ -281,26 +281,28 @@ public class HlsSessionWorker : IHlsSessionWorker } } - private async Task GetPtsOffset(IMediator mediator, string channelNumber, CancellationToken cancellationToken) + private static async Task GetPtsOffset(IMediator mediator, string channelNumber, CancellationToken cancellationToken) { - var directory = new DirectoryInfo(Path.Combine(FileSystemLayout.TranscodeFolder, channelNumber)); - Option lastSegment = - Optional(directory.GetFiles("*.ts").OrderByDescending(f => f.Name).FirstOrDefault()); - - long result = 0; - foreach (FileInfo segment in lastSegment) + await Slim.WaitAsync(cancellationToken); + try { + long result = 0; + Either queryResult = await mediator.Send( - new GetLastPtsDuration(segment.FullName), + new GetLastPtsDuration(channelNumber), cancellationToken); - foreach (PtsAndDuration ptsAndDuration in queryResult.RightToSeq()) + foreach ((long pts, long duration) in queryResult.RightToSeq()) { - result = ptsAndDuration.Pts + ptsAndDuration.Duration; + result = pts + duration; } - } - return result; + return result; + } + finally + { + Slim.Release(); + } } private async Task GetWorkAheadLimit() diff --git a/ErsatzTV.Application/Streaming/Queries/GetLastPtsDuration.cs b/ErsatzTV.Application/Streaming/Queries/GetLastPtsDuration.cs index 8a424b20e..9f854d8be 100644 --- a/ErsatzTV.Application/Streaming/Queries/GetLastPtsDuration.cs +++ b/ErsatzTV.Application/Streaming/Queries/GetLastPtsDuration.cs @@ -2,4 +2,4 @@ namespace ErsatzTV.Application.Streaming; -public record GetLastPtsDuration(string FileName) : IRequest>; +public record GetLastPtsDuration(string ChannelNumber) : IRequest>; diff --git a/ErsatzTV.Application/Streaming/Queries/GetLastPtsDurationHandler.cs b/ErsatzTV.Application/Streaming/Queries/GetLastPtsDurationHandler.cs index 71348b0d5..c51bf9b78 100644 --- a/ErsatzTV.Application/Streaming/Queries/GetLastPtsDurationHandler.cs +++ b/ErsatzTV.Application/Streaming/Queries/GetLastPtsDurationHandler.cs @@ -1,18 +1,37 @@ using System.Diagnostics; using System.Text; +using Bugsnag; using ErsatzTV.Core; using ErsatzTV.Core.Domain; +using ErsatzTV.Core.FFmpeg; +using ErsatzTV.Core.Interfaces.FFmpeg; +using ErsatzTV.Core.Interfaces.Metadata; using ErsatzTV.Core.Interfaces.Repositories; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; namespace ErsatzTV.Application.Streaming; public class GetLastPtsDurationHandler : IRequestHandler> { + private readonly IClient _client; + private readonly ILocalFileSystem _localFileSystem; + private readonly ITempFilePool _tempFilePool; private readonly IConfigElementRepository _configElementRepository; + private readonly ILogger _logger; - public GetLastPtsDurationHandler(IConfigElementRepository configElementRepository) + public GetLastPtsDurationHandler( + IClient client, + ILocalFileSystem localFileSystem, + ITempFilePool tempFilePool, + IConfigElementRepository configElementRepository, + ILogger logger) { + _client = client; + _localFileSystem = localFileSystem; + _tempFilePool = tempFilePool; _configElementRepository = configElementRepository; + _logger = logger; } public async Task> Handle( @@ -21,61 +40,123 @@ public class GetLastPtsDurationHandler : IRequestHandler validation = await Validate(request); return await validation.Match( - Handle, + parameters => Handle(parameters, cancellationToken), error => Task.FromResult>(error.Join())); } private async Task> Validate(GetLastPtsDuration request) => - await ValidateFFprobePath() - .MapT( - ffprobePath => new RequestParameters( - request.FileName, - ffprobePath)); + await ValidateFFprobePath().MapT(ffprobePath => new RequestParameters(request.ChannelNumber, ffprobePath)); - private async Task> Handle(RequestParameters parameters) + private async Task> Handle( + RequestParameters parameters, + CancellationToken cancellationToken) { - var startInfo = new ProcessStartInfo + Option maybeLastSegment = GetLastSegment(parameters.ChannelNumber); + foreach (FileInfo segment in maybeLastSegment) { - FileName = parameters.FFprobePath, - RedirectStandardOutput = true, - RedirectStandardError = true, - UseShellExecute = false, - StandardOutputEncoding = Encoding.UTF8, - StandardErrorEncoding = Encoding.UTF8 - }; - - startInfo.ArgumentList.Add("-v"); - startInfo.ArgumentList.Add("0"); - startInfo.ArgumentList.Add("-show_entries"); - startInfo.ArgumentList.Add("packet=pts,duration"); - startInfo.ArgumentList.Add("-of"); - startInfo.ArgumentList.Add("compact=p=0:nk=1"); - startInfo.ArgumentList.Add("-read_intervals"); - startInfo.ArgumentList.Add("-999999"); - startInfo.ArgumentList.Add(parameters.FileName); - - var probe = new Process - { - StartInfo = startInfo - }; - - probe.Start(); - return await probe.StandardOutput.ReadToEndAsync().MapAsync>( - async output => + var startInfo = new ProcessStartInfo { - await probe.WaitForExitAsync(); - return probe.ExitCode == 0 - ? PtsAndDuration.From(output.Split("\n").Filter(s => !string.IsNullOrWhiteSpace(s)).Last().Trim()) - : BaseError.New($"FFprobe at {parameters.FFprobePath} exited with code {probe.ExitCode}"); - }); + FileName = parameters.FFprobePath, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + StandardOutputEncoding = Encoding.UTF8, + StandardErrorEncoding = Encoding.UTF8 + }; + + startInfo.ArgumentList.Add("-v"); + startInfo.ArgumentList.Add("0"); + startInfo.ArgumentList.Add("-show_entries"); + startInfo.ArgumentList.Add("packet=pts,duration"); + startInfo.ArgumentList.Add("-of"); + startInfo.ArgumentList.Add("compact=p=0:nk=1"); + startInfo.ArgumentList.Add("-read_intervals"); + startInfo.ArgumentList.Add("-999999"); + startInfo.ArgumentList.Add(segment.FullName); + + var probe = new Process + { + StartInfo = startInfo + }; + + probe.Start(); + string output = await probe.StandardOutput.ReadToEndAsync(); + await probe.WaitForExitAsync(cancellationToken); + if (probe.ExitCode != 0) + { + return BaseError.New($"FFprobe at {parameters.FFprobePath} exited with code {probe.ExitCode}"); + } + + try + { + string[] lines = output.Split("\n"); + IEnumerable nonEmptyLines = lines.Filter(s => !string.IsNullOrWhiteSpace(s)).Map(l => l.Trim()); + return PtsAndDuration.From(nonEmptyLines.Last()); + } + catch (Exception ex) + { + _client.Notify(ex); + await SaveTroubleshootingData(parameters.ChannelNumber, output); + } + } + + return BaseError.New($"Failed to determine last pts duration for channel {parameters.ChannelNumber}"); } + private static Option GetLastSegment(string channelNumber) + { + var directory = new DirectoryInfo(Path.Combine(FileSystemLayout.TranscodeFolder, channelNumber)); + return Optional(directory.GetFiles("*.ts").OrderByDescending(f => f.Name).FirstOrDefault()); + } + private Task> ValidateFFprobePath() => _configElementRepository.GetValue(ConfigElementKey.FFprobePath) .FilterT(File.Exists) - .Map( - ffprobePath => - ffprobePath.ToValidation("FFprobe path does not exist on the file system")); + .Map(ffprobePath => ffprobePath.ToValidation("FFprobe path does not exist on the file system")); - private record RequestParameters(string FileName, string FFprobePath); + private async Task SaveTroubleshootingData(string channelNumber, string output) + { + try + { + var directory = new DirectoryInfo(Path.Combine(FileSystemLayout.TranscodeFolder, channelNumber)); + FileInfo[] allFiles = directory.GetFiles(); + + string playlistFileName = Path.Combine(FileSystemLayout.TranscodeFolder, channelNumber, "live.m3u8"); + string playlistContents = string.Empty; + if (_localFileSystem.FileExists(playlistFileName)) + { + playlistContents = await File.ReadAllTextAsync(playlistFileName); + } + + var data = new TroubleshootingData(allFiles, playlistContents, output); + string serialized = data.Serialize(); + + string file = _tempFilePool.GetNextTempFile(TempFileCategory.BadTranscodeFolder); + await File.WriteAllTextAsync(file, serialized); + + _logger.LogWarning("Transcode folder is in bad state; troubleshooting info saved to {File}", file); + } + catch (Exception ex) + { + _client.Notify(ex); + } + } + + private record RequestParameters(string ChannelNumber, string FFprobePath); + + private record TroubleshootingData(IEnumerable Files, string Playlist, string ProbeOutput) + { + private record FileData(string FileName, long Bytes, DateTime LastWriteTimeUtc); + private record InternalData(List Files, string EncodedPlaylist, string EncodedProbeOutput); + + public string Serialize() + { + var data = new InternalData( + Files.Map(f => new FileData(f.FullName, f.Length, f.LastWriteTimeUtc)).ToList(), + Convert.ToBase64String(Encoding.UTF8.GetBytes(Playlist)), + Convert.ToBase64String(Encoding.UTF8.GetBytes(ProbeOutput))); + + return JsonConvert.SerializeObject(data); + } + } } diff --git a/ErsatzTV.Core/FFmpeg/TempFileCategory.cs b/ErsatzTV.Core/FFmpeg/TempFileCategory.cs index 33d60573b..fc3021740 100644 --- a/ErsatzTV.Core/FFmpeg/TempFileCategory.cs +++ b/ErsatzTV.Core/FFmpeg/TempFileCategory.cs @@ -7,5 +7,6 @@ public enum TempFileCategory CoverArt = 2, CachedArtwork = 3, + BadTranscodeFolder = 98, BadPlaylist = 99 } \ No newline at end of file