using System.ComponentModel.DataAnnotations; using System.IO.Abstractions; using System.Text.Json; using System.Text.Json.Serialization; using System.Threading.Channels; using ErsatzTV.Application; using ErsatzTV.Application.MediaItems; using ErsatzTV.Application.Troubleshooting; using ErsatzTV.Application.Troubleshooting.Queries; using ErsatzTV.Controllers.Api.Requests; using ErsatzTV.Core; using ErsatzTV.Core.Api.Troubleshooting; using ErsatzTV.Application.Channels; using ErsatzTV.Core.Domain; using ErsatzTV.Core.Interfaces.FFmpeg; using ErsatzTV.Core.Interfaces.Locking; using ErsatzTV.Core.Interfaces.Repositories; using ErsatzTV.Core.Interfaces.Troubleshooting; using ErsatzTV.Core.Troubleshooting; using ErsatzTV.Extensions; using MediatR; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Serilog.Context; namespace ErsatzTV.Controllers.Api; [ApiController] public class TroubleshootController( ChannelWriter channelWriter, IFileSystem fileSystem, IConfigElementRepository configElementRepository, ITroubleshootingNotifier notifier, IEntityLocker entityLocker, ITroubleshootingPlaybackStatusStore statusStore, IMediator mediator) : ControllerBase { private static readonly JsonSerializerOptions GeneralJsonOptions = new() { Converters = { new JsonStringEnumConverter() }, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, WriteIndented = true }; [HttpGet("api/troubleshoot/info", Name = "GetTroubleshootingInfo")] [Tags("Troubleshooting")] [EndpointSummary("Get troubleshooting diagnostic info")] [EndpointGroupName("general")] [ProducesResponseType(typeof(TroubleshootingInfoResponseModel), StatusCodes.Status200OK)] public async Task GetInfo(CancellationToken cancellationToken) { TroubleshootingInfo info = await mediator.Send(new GetTroubleshootingInfo(), cancellationToken); // mirrors the "General" tab JSON blob built by Pages/Troubleshooting/Troubleshooting.razor string generalJson = JsonSerializer.Serialize( new { info.Version, Environment = info.Environment.OrderBy(x => x.Key).ToDictionary(x => x.Key, x => x.Value), info.Cpus, info.VideoControllers, info.Health, info.FFmpegSettings, AviSynth = new { Demuxer = info.AviSynthDemuxer, Installed = info.AviSynthInstalled }, info.Channels, info.FFmpegProfiles }, GeneralJsonOptions); return new TroubleshootingInfoResponseModel( generalJson, info.NvidiaCapabilities, info.QsvCapabilities, info.VaapiCapabilities, info.VideoToolboxCapabilities); } [HttpPost("api/troubleshoot/validate-schedule", Name = "ValidateSequentialSchedule")] [Tags("Troubleshooting")] [EndpointSummary("Validate a sequential schedule YAML document")] [EndpointDescription( "Validates a sequential-schedule YAML string against the full (or import) schema. Returns whether it is " + "valid, any validation messages, and the JSON conversion of the YAML. Parse/validator errors are reported " + "as messages (IsValid=false), never as a 500.")] [EndpointGroupName("general")] [ProducesResponseType(typeof(ValidateSequentialScheduleResponseModel), StatusCodes.Status200OK)] [ProducesResponseType(typeof(ProblemDetails), StatusCodes.Status400BadRequest)] public async Task ValidateSchedule( [Required] [FromBody] ValidateSequentialScheduleRequest request, CancellationToken cancellationToken) { if (string.IsNullOrWhiteSpace(request.Yaml)) { return new BadRequestObjectResult( new ProblemDetails { Status = StatusCodes.Status400BadRequest, Title = "Validation failed", Detail = "[Yaml] must not be empty" }); } ValidateSequentialScheduleViewModel result = await mediator.Send(request.ToQuery(), cancellationToken); return new OkObjectResult( new ValidateSequentialScheduleResponseModel(result.IsValid, result.Messages, result.Json)); } [HttpHead("api/troubleshoot/playback.m3u8")] [HttpGet("api/troubleshoot/playback.m3u8")] [Tags("Troubleshooting")] [EndpointSummary("Start a troubleshooting playback session")] [EndpointGroupName("general")] [ProducesResponseType(typeof(ProblemDetails), StatusCodes.Status409Conflict)] public async Task TroubleshootPlayback( [FromQuery] int mediaItem, [FromQuery] int channel, [FromQuery] int ffmpegProfile, [FromQuery] StreamingMode streamingMode, [FromQuery] List watermark, [FromQuery] List graphicsElement, [FromQuery] string streamSelector, [FromQuery] int? subtitleId, [FromQuery] int seekSeconds, [FromQuery] DateTimeOffset? start, CancellationToken cancellationToken) { var sessionId = Guid.NewGuid(); using var logContext = LogContext.PushProperty(InMemoryLogService.CorrelationIdKey, sessionId); // acquiredLock: the handler took the troubleshooting lock on a successful Prepare. // startEnqueued: we handed off to StartTroubleshootingPlayback, which becomes the lock's releaser. var acquiredLock = false; var startEnqueued = false; try { // fast lock-conflict signal so the client gets a 409 (not a 404) while another // troubleshooting session is active; the atomic acquire in the handler is the real guard if (entityLocker.IsTroubleshootingPlaybackLocked()) { return ApiResults.ConflictProblem( "Troubleshooting playback in progress", "Another troubleshooting playback session is currently running. Try again once it completes."); } Option ss = seekSeconds > 0 ? seekSeconds : Option.None; Either result = await mediator.Send( new PrepareTroubleshootingPlayback( sessionId, streamingMode, mediaItem, channel, ffmpegProfile, streamSelector, watermark, graphicsElement, subtitleId, ss, Optional(start)), cancellationToken); if (result.IsLeft) { return NotFound(); } // Prepare returned a process, so the handler holds the troubleshooting lock now acquiredLock = true; foreach (PlayoutItemResult playoutItemResult in result.RightToSeq()) { Either maybeMediaInfo = await mediator.Send( new GetMediaItemInfo(await playoutItemResult.MediaItemId.IfNoneAsync(0)), cancellationToken); try { TroubleshootingInfo troubleshootingInfo = await mediator.Send( new GetTroubleshootingInfo(), cancellationToken); // filter ffmpeg profiles troubleshootingInfo.FFmpegProfiles.RemoveAll(p => p.Id != ffmpegProfile); // filter watermarks troubleshootingInfo.Watermarks.RemoveAll(p => !watermark.Contains(p.Id)); await channelWriter.WriteAsync( new StartTroubleshootingPlayback( sessionId, streamSelector, playoutItemResult, maybeMediaInfo.ToOption(), troubleshootingInfo), cancellationToken); // StartTroubleshootingPlayback is now responsible for releasing the lock in its finally startEnqueued = true; string playlistFile = Path.Combine(FileSystemLayout.TranscodeTroubleshootingFolder, "live.m3u8"); while (!fileSystem.File.Exists(playlistFile)) { await Task.Delay(TimeSpan.FromMilliseconds(100), cancellationToken); if (cancellationToken.IsCancellationRequested || notifier.IsFailed(sessionId)) { break; } } int initialSegmentCount = await configElementRepository .GetValue(ConfigElementKey.FFmpegInitialSegmentCount, cancellationToken) .Map(maybeCount => maybeCount.Match(c => c, () => 1)); initialSegmentCount = Math.Max(initialSegmentCount, 2); bool hasSegments = false; while (!hasSegments) { await Task.Delay(TimeSpan.FromMilliseconds(100), cancellationToken); string[] segmentFiles = streamingMode switch { // StreamingMode.HttpLiveStreamingSegmenter => Directory.GetFiles( // FileSystemLayout.TranscodeTroubleshootingFolder, // "*.m4s"), _ => Directory.GetFiles(FileSystemLayout.TranscodeTroubleshootingFolder, "*.ts") }; if (segmentFiles.Length >= initialSegmentCount) { hasSegments = true; } } if (!notifier.IsFailed(sessionId)) { return Redirect("~/iptv/session/.troubleshooting/live.m3u8"); } } finally { notifier.RemoveSession(sessionId); } } } catch (Exception) { // do nothing } finally { // the handler acquired the lock but we never handed off to the worker // (cancellation/exception in the window before enqueue) -> release it so it doesn't leak if (acquiredLock && !startEnqueued) { entityLocker.UnlockTroubleshootingPlayback(); } } return NotFound(); } [HttpHead("api/troubleshoot/playback/archive")] [HttpGet("api/troubleshoot/playback/archive")] [Tags("Troubleshooting")] [EndpointSummary("Download the last troubleshooting playback session archive")] [EndpointGroupName("general")] public async Task TroubleshootPlaybackArchive(CancellationToken cancellationToken) { Option maybeArchivePath = await mediator.Send(new ArchiveTroubleshootingResults(), cancellationToken); foreach (string archivePath in maybeArchivePath) { var fs = new FileStream( archivePath, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, FileOptions.DeleteOnClose); return File( fs, "application/zip", $"ersatztv-troubleshooting-{DateTimeOffset.Now.ToUnixTimeSeconds()}.zip"); } return NotFound(); } [HttpHead("api/troubleshoot/playback/sample/{mediaItemId:int}")] [HttpGet("api/troubleshoot/playback/sample/{mediaItemId:int}")] [Tags("Troubleshooting")] [EndpointSummary("Download a media sample archive for troubleshooting")] [EndpointGroupName("general")] public async Task TroubleshootPlaybackSample(int mediaItemId, CancellationToken cancellationToken) { Option maybeArchivePath = await mediator.Send(new ArchiveMediaSample(mediaItemId), cancellationToken); foreach (string archivePath in maybeArchivePath) { var fs = new FileStream( archivePath, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, FileOptions.DeleteOnClose); return File( fs, "application/zip", $"ersatztv-media-sample-{DateTimeOffset.Now.ToUnixTimeSeconds()}.zip"); } return NotFound(); } [HttpGet("api/troubleshoot/playback/stream-selectors", Name = "GetTroubleshootingStreamSelectors")] [Tags("Troubleshooting")] [EndpointSummary("List available channel stream selectors")] [EndpointGroupName("general")] [ProducesResponseType(typeof(List), StatusCodes.Status200OK)] public async Task> GetStreamSelectors(CancellationToken cancellationToken) => await mediator.Send(new GetChannelStreamSelectors(), cancellationToken); [HttpGet("api/troubleshoot/playback/subtitles/{mediaItemId:int}", Name = "GetTroubleshootingSubtitles")] [Tags("Troubleshooting")] [EndpointSummary("List selectable subtitle streams for a media item")] [EndpointDescription( "Returns the subtitle streams that can be burned in for a troubleshooting playback. Each item's id is the " + "value to pass back as the playback.m3u8 endpoint's subtitleId query parameter.")] [EndpointGroupName("general")] [ProducesResponseType(typeof(List), StatusCodes.Status200OK)] [ProducesResponseType(typeof(ProblemDetails), StatusCodes.Status404NotFound)] public async Task GetSubtitles(int mediaItemId, CancellationToken cancellationToken) { Either maybeMediaItem = await mediator.Send(new GetMediaItemInfo(mediaItemId), cancellationToken); if (maybeMediaItem.IsLeft) { return ApiResults.NotFoundProblem(); } List subtitles = await mediator.Send(new GetTroubleshootingSubtitles(mediaItemId), cancellationToken); return new OkObjectResult( subtitles .Map(s => new TroubleshootingSubtitleResponseModel(s.Id, s.Language, s.Title, s.Codec)) .ToList()); } [HttpGet("api/troubleshoot/playback/status", Name = "GetTroubleshootingPlaybackStatus")] [Tags("Troubleshooting")] [EndpointSummary("Get the status of the current or last troubleshooting playback session")] [EndpointDescription( "Reports whether a troubleshooting playback is idle, running, completed, or failed, along with the last " + "session's ffmpeg exit code, playback speed, and a tail (last 500 lines) of its log output.")] [EndpointGroupName("general")] [ProducesResponseType(typeof(TroubleshootingPlaybackStatusResponseModel), StatusCodes.Status200OK)] public async Task GetPlaybackStatus(CancellationToken cancellationToken) { bool running = entityLocker.IsTroubleshootingPlaybackLocked(); Option maybeResult = statusStore.CurrentResult; string state = "idle"; int? exitCode = null; double? speed = null; if (running) { state = "running"; } foreach (TroubleshootingPlaybackResult result in maybeResult) { exitCode = result.ExitCode; speed = result.Speed.MatchUnsafe(v => (double?)v, () => null); if (!running) { state = result.ExitCode == 0 ? "completed" : "failed"; } } string logs = await ReadTroubleshootingLogTail(cancellationToken); return new TroubleshootingPlaybackStatusResponseModel(state, exitCode, speed, logs); } private async Task ReadTroubleshootingLogTail(CancellationToken cancellationToken) { const int MaxLines = 500; string logFile = Path.Combine(FileSystemLayout.TranscodeTroubleshootingFolder, "logs.txt"); if (!fileSystem.File.Exists(logFile)) { return null; } string[] lines = await fileSystem.File.ReadAllLinesAsync(logFile, cancellationToken); if (lines.Length > MaxLines) { lines = lines[^MaxLines..]; } return string.Join(Environment.NewLine, lines); } }